diff --git "a/qformer2_128/wandb/run-20240312_142853-zo3nkbji/logs/debug-internal.log" "b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/logs/debug-internal.log" @@ -0,0 +1,42131 @@ +2024-03-12 14:28:53,689 INFO StreamThr :1827008 [internal.py:wandb_internal():86] W&B internal server running at pid: 1827008, started at: 2024-03-12 14:28:53.687253 +2024-03-12 14:28:53,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status +2024-03-12 14:28:53,701 INFO WriterThread:1827008 [datastore.py:open_for_write():87] open: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/run-zo3nkbji.wandb +2024-03-12 14:28:53,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: header +2024-03-12 14:28:53,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: run +2024-03-12 14:28:54,216 INFO SenderThread:1827008 [dir_watcher.py:__init__():211] watching files in: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files +2024-03-12 14:28:54,216 INFO SenderThread:1827008 [sender.py:_start_run_threads():1136] run started: zo3nkbji with start time 1710253733.688548 +2024-03-12 14:28:54,230 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: check_version +2024-03-12 14:28:54,230 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: check_version +2024-03-12 14:28:54,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: run_start +2024-03-12 14:28:54,290 DEBUG HandlerThread:1827008 [system_info.py:__init__():27] System info init +2024-03-12 14:28:54,290 DEBUG HandlerThread:1827008 [system_info.py:__init__():42] System info init done +2024-03-12 14:28:54,290 INFO HandlerThread:1827008 [system_monitor.py:start():194] Starting system monitor +2024-03-12 14:28:54,290 INFO SystemMonitor:1827008 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-12 14:28:54,291 INFO HandlerThread:1827008 [system_monitor.py:probe():214] Collecting system info +2024-03-12 14:28:54,291 INFO SystemMonitor:1827008 [interfaces.py:start():190] Started cpu monitoring +2024-03-12 14:28:54,292 INFO SystemMonitor:1827008 [interfaces.py:start():190] Started disk monitoring +2024-03-12 14:28:54,292 INFO SystemMonitor:1827008 [interfaces.py:start():190] Started gpu monitoring +2024-03-12 14:28:54,293 INFO SystemMonitor:1827008 [interfaces.py:start():190] Started memory monitoring +2024-03-12 14:28:54,294 INFO SystemMonitor:1827008 [interfaces.py:start():190] Started network monitoring +2024-03-12 14:28:54,394 DEBUG HandlerThread:1827008 [system_info.py:probe():151] Probing system +2024-03-12 14:28:54,398 DEBUG HandlerThread:1827008 [system_info.py:_probe_git():136] Probing git +2024-03-12 14:28:54,419 DEBUG HandlerThread:1827008 [system_info.py:_probe_git():144] Probing git done +2024-03-12 14:28:54,419 DEBUG HandlerThread:1827008 [system_info.py:probe():199] Probing system done +2024-03-12 14:28:54,419 DEBUG HandlerThread:1827008 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-12T14:28:54.394423', 'startedAt': '2024-03-12T14:28:53.639298', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'one-stage+7b_qformer2_128', '--run_id', 'qformer2_128', '--model.arch_specifier', 'qformer2_128'), 'state': 'running', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms.git', 'commit': 'b1c33c284c6e76d908c29e5545b00f5b8b4c601f'}, 'email': 'tobiaslee@qq.com', 'root': '/home/lilei/prismatic-vlms', 'host': 'compute-permanent-node-815', 'username': 'lilei', 'executable': '/home/lilei/anaconda3/envs/pvlm/bin/python', 'cpu_count': 128, 'cpu_count_logical': 255, 'cpu_freq': {'current': 2571.1995686274513, 'min': 1500.0, 'max': 2550.0}, 'cpu_freq_per_core': [{'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.918, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.707, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.201, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.098, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.2, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3245.15, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.386, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.237, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.479, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}], 'disk': {'/': {'total': 242.1086883544922, 'used': 55.302242279052734}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.682991027832}} +2024-03-12 14:28:54,420 INFO HandlerThread:1827008 [system_monitor.py:probe():224] Finished collecting system info +2024-03-12 14:28:54,420 INFO HandlerThread:1827008 [system_monitor.py:probe():227] Publishing system info +2024-03-12 14:28:54,420 DEBUG HandlerThread:1827008 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-03-12 14:28:55,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/conda-environment.yaml +2024-03-12 14:28:59,025 DEBUG HandlerThread:1827008 [system_info.py:_save_conda():220] Saving conda packages done +2024-03-12 14:28:59,034 INFO HandlerThread:1827008 [system_monitor.py:probe():229] Finished publishing system info +2024-03-12 14:28:59,040 DEBUG SenderThread:1827008 [sender.py:send():382] send: files +2024-03-12 14:28:59,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-12 14:28:59,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:28:59,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: python_packages +2024-03-12 14:28:59,051 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: python_packages +2024-03-12 14:28:59,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:28:59,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:28:59,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:28:59,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/conda-environment.yaml +2024-03-12 14:28:59,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-metadata.json +2024-03-12 14:28:59,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/requirements.txt +2024-03-12 14:28:59,296 DEBUG SenderThread:1827008 [sender.py:send():382] send: telemetry +2024-03-12 14:29:00,132 INFO wandb-upload_0:1827008 [upload_job.py:push():131] Uploaded file /tmp/tmpeemnmoqdwandb/r7dv6dtt-wandb-metadata.json +2024-03-12 14:29:00,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:02,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:04,454 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:05,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:09,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:14,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:14,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:14,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:15,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:20,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:24,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:26,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:27,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/config.yaml +2024-03-12 14:29:27,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:29,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:29,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:29,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:31,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:32,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:32,163 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:32,164 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:32,167 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:32,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:29:35,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:35,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:35,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:35,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:35,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:36,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:29:36,815 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:37,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:39,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:39,600 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:39,601 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:39,606 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:40,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:29:41,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:42,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:44,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:44,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:44,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:44,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:44,457 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:44,458 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:44,463 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:45,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:29:47,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:48,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:48,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:48,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:48,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:48,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:49,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:29:51,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:53,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:53,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:53,138 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:53,143 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:53,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:29:54,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:54,294 DEBUG SystemMonitor:1827008 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-12 14:29:54,297 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:29:55,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:58,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:58,740 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:58,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:58,746 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:59,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:59,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:59,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:59,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:59,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:01,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:03,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:03,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:03,135 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:03,140 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:03,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:05,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:05,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:10,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:10,014 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:10,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:10,019 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:10,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:11,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:11,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:14,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:14,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:14,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:14,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:14,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:14,349 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:14,354 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:15,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:16,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:17,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:18,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:18,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:18,298 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:18,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:18,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:21,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:22,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:22,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:22,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:22,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:22,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:23,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:24,300 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:30:25,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:26,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:26,254 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:26,254 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:26,259 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:26,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:28,260 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:29,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:29,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:29,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:29,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:31,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:31,538 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:31,538 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:31,543 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:32,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:33,321 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:33,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:37,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:37,002 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:37,003 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:37,008 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:37,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:39,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:39,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:41,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:41,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:41,385 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:41,390 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:42,328 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:43,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:44,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:44,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:44,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:44,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:46,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:46,229 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:46,229 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:46,234 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:46,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:49,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:50,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:50,102 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:50,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:50,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:50,108 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:50,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:53,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:53,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:53,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:53,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:53,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:54,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:30:54,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:55,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:55,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:59,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:59,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:59,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:59,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:59,346 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:59,346 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:59,351 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:59,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:00,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:01,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:03,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:03,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:03,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:03,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:04,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:05,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:05,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:10,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:12,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:12,028 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:12,029 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:12,034 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:12,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:13,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:14,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:14,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:14,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:15,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:15,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:15,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:15,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:15,750 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:16,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:17,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:20,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:20,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:20,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:20,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:20,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:21,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:21,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:24,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:31:26,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:29,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:29,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:29,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:29,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:29,395 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:29,395 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:29,400 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:30,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:31,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:31,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:32,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:32,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:32,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:32,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:33,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:35,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:36,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:36,571 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:36,571 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:36,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:36,576 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:37,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:39,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:41,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:41,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:41,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:41,813 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:41,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:42,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:43,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:44,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:44,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:44,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:47,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:47,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:47,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:47,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:47,115 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:47,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:49,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:51,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:51,589 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:51,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:51,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:52,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:52,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:53,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:54,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:31:57,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:57,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:57,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:57,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:57,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:58,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:59,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:59,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:59,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:02,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:02,642 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:02,642 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:02,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:03,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:03,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:05,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:06,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:06,780 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:06,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:06,785 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:07,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:08,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:09,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:11,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:11,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:11,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:11,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:11,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:13,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:14,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:14,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:14,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:14,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:16,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:16,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:16,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:16,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:17,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:19,431 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:20,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:24,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:24,281 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:24,281 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:24,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:24,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:32:24,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:25,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:27,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:28,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:28,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:28,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:28,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:28,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:29,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:29,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:29,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:29,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:30,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:31,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:31,786 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:31,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:31,792 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:32,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:33,447 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:35,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:37,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:37,422 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:37,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:37,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:37,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:39,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:40,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:40,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:40,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:40,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:41,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:41,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:43,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:44,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:44,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:44,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:44,354 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:44,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:44,359 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:44,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:47,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:47,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:49,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:49,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:49,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:49,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:50,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:51,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:52,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:54,319 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:32:54,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:54,490 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:54,490 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:54,495 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:55,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:57,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:58,496 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:59,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:59,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:02,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:02,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:02,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:02,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:02,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:04,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:05,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:09,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:12,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:12,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:12,627 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:12,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:13,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:14,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:14,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:14,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:15,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:15,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:17,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:17,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:17,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:17,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:18,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:19,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:20,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:23,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:23,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:23,246 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:23,251 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:23,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:24,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:33:25,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:26,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:27,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:27,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:27,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:27,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:27,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:29,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:29,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:29,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:29,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:31,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:31,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:31,813 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:31,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:31,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:32,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:33,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:35,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:35,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:35,807 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:35,812 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:36,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:37,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:37,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:40,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:40,704 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:40,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:40,709 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:41,511 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:43,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:43,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:44,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:44,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:46,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:46,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:46,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:46,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:47,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:48,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:49,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:51,835 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:51,836 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:51,836 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:51,841 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:52,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:53,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:53,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:54,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:33:57,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:57,281 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:57,281 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:57,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:57,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:59,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:59,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:59,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:59,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:03,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:03,201 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:03,202 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:03,206 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:03,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:05,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:05,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:07,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:07,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:07,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:07,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:07,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:09,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:10,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:14,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:14,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:14,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:14,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:14,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:14,256 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:14,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:14,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:15,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:16,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:19,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:19,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:19,997 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:20,002 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:20,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:21,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:22,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:23,870 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:23,871 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:23,871 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:23,876 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:24,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:34:24,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:25,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:27,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:28,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:28,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:28,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:28,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:28,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:29,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:29,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:29,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:29,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:32,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:32,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:32,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:32,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:32,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:33,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:33,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:36,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:36,121 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:36,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:36,126 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:36,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:37,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:38,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:43,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:43,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:43,679 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:43,679 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:43,684 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:44,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:44,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:44,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:45,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:48,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:48,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:48,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:48,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:48,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:49,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:51,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:53,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:53,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:53,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:53,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:53,580 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:54,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:54,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:34:55,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:56,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:56,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:56,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:56,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:57,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:59,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:59,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:59,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:59,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:01,842 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:01,843 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:01,843 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:01,848 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:02,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:03,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:04,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:05,520 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:05,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:05,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:05,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:05,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:07,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:09,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:09,436 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:09,436 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:09,441 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:09,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:10,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:11,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:13,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:13,472 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:13,472 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:13,478 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:13,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:14,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:14,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:14,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:15,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:16,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:17,034 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:17,035 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:17,035 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:17,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:17,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:19,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:20,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:20,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:20,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:20,916 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:21,607 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:21,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:23,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:24,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:35:24,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:24,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:24,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:24,895 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:25,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:27,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:27,896 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:29,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:29,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:29,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:29,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:29,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:29,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:29,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:30,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:31,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:33,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:34,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:34,336 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:34,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:34,342 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:34,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:35,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:37,875 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:37,876 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:37,876 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:37,881 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:38,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:38,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:39,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:41,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:41,456 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:41,457 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:41,461 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:41,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:43,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:44,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:44,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:44,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:44,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:45,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:45,392 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:45,392 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:45,397 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:45,635 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:47,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:49,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:50,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:51,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:51,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:51,005 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:51,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:53,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:54,330 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:35:55,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:55,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:55,935 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:55,935 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:55,940 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:56,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:57,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:59,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:59,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:59,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:59,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:59,734 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:59,739 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:00,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:00,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:01,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:05,539 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:05,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:05,541 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:05,545 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:05,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:06,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:07,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:10,831 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:10,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:10,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:10,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:11,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:11,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:13,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:14,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:14,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:14,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:14,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:14,791 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:14,792 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:14,796 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:15,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:17,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:17,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:22,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:24,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:24,106 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:24,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:24,111 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:24,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:36:24,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:25,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:27,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:27,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:27,990 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:27,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:27,995 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:28,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:29,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:29,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:29,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:29,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:33,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:33,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:33,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:33,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:33,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:34,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:35,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:37,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:37,792 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:37,792 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:37,797 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:38,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:38,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:39,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:43,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:43,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:43,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:43,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:43,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:44,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:45,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:49,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:49,068 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:49,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:49,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:49,073 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:49,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:51,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:53,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:53,198 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:53,198 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:53,203 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:53,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:54,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:54,334 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:36:55,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:57,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:57,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:57,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:57,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:57,708 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:59,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:59,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:59,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:59,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:01,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:01,129 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:01,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:01,134 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:01,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:03,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:05,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:05,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:05,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:05,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:05,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:05,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:07,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:10,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:10,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:10,507 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:10,507 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:10,512 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:10,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:13,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:14,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:14,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:14,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:15,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:15,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:15,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:15,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:15,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:17,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:19,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:19,060 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:19,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:19,065 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:19,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:21,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:21,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:24,336 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:37:26,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:27,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:27,344 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:27,345 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:27,350 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:27,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:29,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:29,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:29,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:31,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:31,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:31,734 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:31,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:31,739 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:31,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:33,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:36,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:36,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:36,980 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:36,980 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:36,985 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:37,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:39,750 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:40,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:40,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:40,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:40,373 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:40,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:41,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:42,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:44,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:48,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:48,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:48,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:48,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:48,802 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:49,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:51,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:52,548 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:52,549 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:52,549 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:52,554 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:52,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:53,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:54,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:37:55,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:55,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:55,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:55,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:55,895 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:56,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:57,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:58,896 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:59,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:59,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:59,650 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:59,650 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:59,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:59,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:01,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:03,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:03,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:03,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:03,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:03,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:04,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:05,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:07,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:07,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:07,628 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:07,633 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:07,779 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:09,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:09,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:12,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:12,162 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:12,163 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:12,168 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:12,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:13,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:14,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:14,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:15,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:15,868 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:15,869 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:15,869 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:15,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:16,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:17,790 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:20,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:21,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:21,532 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:21,532 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:21,537 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:21,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:23,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:24,340 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:38:26,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:26,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:26,716 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:26,716 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:26,721 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:26,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:29,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:29,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:29,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:32,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:32,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:32,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:32,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:32,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:32,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:33,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:37,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:41,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:41,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:41,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:41,813 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:41,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:42,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:43,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:44,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:46,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:46,402 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:46,403 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:46,407 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:46,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:47,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:48,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:52,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:52,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:52,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:52,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:52,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:53,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:54,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:38:55,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:57,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:57,240 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:57,240 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:57,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:57,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:59,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:59,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:59,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:01,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:01,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:01,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:01,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:01,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:03,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:04,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:05,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:05,102 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:05,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:05,108 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:05,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:07,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:09,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:09,658 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:09,658 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:09,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:09,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:09,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:11,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:14,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:14,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:15,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:15,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:15,704 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:15,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:15,709 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:15,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:17,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:19,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:19,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:19,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:19,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:19,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:20,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:21,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:23,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:23,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:23,360 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:23,365 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:23,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:24,344 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:39:25,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:26,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:27,548 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:27,549 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:27,550 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:27,555 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:27,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:29,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:29,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:29,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:31,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:31,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:31,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:31,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:31,709 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:31,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:33,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:36,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:38,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:38,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:38,173 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:38,178 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:38,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:39,883 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:41,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:41,914 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:41,914 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:41,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:41,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:42,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:43,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:44,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:46,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:46,061 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:46,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:46,067 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:46,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:47,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:47,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:49,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:49,966 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:49,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:49,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:50,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:51,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:52,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:54,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:39:54,984 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:54,985 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:54,985 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:54,990 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:55,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:57,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:57,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:58,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:58,597 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:58,597 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:58,602 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:58,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:59,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:59,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:59,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:02,427 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:02,428 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:02,428 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:02,434 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:02,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:03,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:03,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:08,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:08,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:08,485 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:08,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:08,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:08,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:09,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:12,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:12,371 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:12,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:12,377 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:12,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:13,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:14,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:14,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:16,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:16,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:16,913 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:16,917 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:16,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:19,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:19,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:20,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:20,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:20,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:20,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:20,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:21,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:24,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:40:25,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:25,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:25,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:25,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:25,317 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:25,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:27,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:28,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:28,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:28,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:28,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:28,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:29,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:29,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:29,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:31,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:32,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:32,803 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:32,803 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:32,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:32,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:35,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:36,231 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:36,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:36,232 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:36,239 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:36,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:37,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:37,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:39,611 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:39,611 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:39,612 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:39,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:39,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:41,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:42,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:44,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:44,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:44,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:48,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:48,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:48,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:48,956 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:48,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:48,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:51,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:54,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:54,357 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:40:56,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:56,117 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:56,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:56,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:56,987 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:57,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:59,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:59,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:04,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:04,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:04,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:04,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:04,706 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:04,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:06,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:09,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:13,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:13,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:13,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:13,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:14,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:14,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:14,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:15,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:16,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:17,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:17,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:17,173 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:17,186 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:18,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:20,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:21,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:21,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:21,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:21,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:21,857 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:22,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:24,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:24,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:41:25,816 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:25,817 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:25,817 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:25,826 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:26,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:26,827 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:28,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:29,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:29,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:29,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:29,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:29,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:29,957 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:29,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:30,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:32,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:32,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:34,799 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:34,800 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:34,800 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:34,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:35,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:37,810 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:38,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:39,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:39,301 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:39,302 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:39,312 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:40,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:42,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:43,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:43,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:43,705 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:43,705 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:43,715 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:44,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:44,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:44,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:44,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:46,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:49,250 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:49,251 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:49,251 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:49,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:49,258 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:50,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:52,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:53,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:53,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:53,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:53,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:54,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:54,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:54,355 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:41:56,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:57,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:57,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:57,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:57,780 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:58,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:59,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:00,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:00,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:05,267 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:06,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:06,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:06,303 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:06,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:07,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:08,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:10,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:10,267 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:10,267 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:10,267 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:10,276 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:11,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:12,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:14,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:14,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:14,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:14,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:14,430 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:14,430 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:14,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:15,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:15,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:16,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:17,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:17,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:17,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:17,963 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:18,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:20,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:20,963 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:22,871 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:22,872 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:22,872 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:22,878 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:23,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:24,357 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:42:26,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:26,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:27,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:27,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:27,144 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:27,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:27,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:29,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:29,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:29,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:30,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:31,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:31,087 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:31,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:31,093 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:31,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:32,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:34,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:35,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:35,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:35,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:35,665 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:36,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:37,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:38,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:41,152 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:41,153 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:41,154 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:41,162 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:41,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:43,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:44,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:44,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:44,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:44,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:45,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:45,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:45,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:45,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:46,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:48,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:48,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:49,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:49,503 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:49,503 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:49,513 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:50,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:52,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:53,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:53,449 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:53,449 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:53,457 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:54,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:54,359 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:42:54,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:56,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:57,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:57,620 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:57,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:57,627 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:58,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:59,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:59,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:00,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:00,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:01,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:01,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:01,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:01,455 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:02,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:04,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:05,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:09,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:09,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:09,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:09,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:10,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:10,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:12,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:14,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:14,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:14,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:14,522 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:14,522 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:14,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:15,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:16,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:16,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:19,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:19,775 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:19,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:19,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:20,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:21,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:22,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:23,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:23,565 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:23,565 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:23,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:24,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:24,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:43:26,280 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:27,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:28,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:28,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:28,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:28,159 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:28,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:29,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:29,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:30,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:31,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:31,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:31,906 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:31,916 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:32,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:32,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:33,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:37,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:41,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:41,030 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:41,031 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:41,038 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:41,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:43,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:43,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:44,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:44,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:45,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:45,603 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:45,603 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:45,611 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:46,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:47,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:48,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:50,236 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:50,237 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:50,237 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:50,247 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:50,536 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:51,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:54,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:54,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:43:54,816 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:54,817 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:54,817 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:54,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:55,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:55,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:59,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:59,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:59,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:59,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:59,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:59,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:59,957 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:59,963 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:00,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:01,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:04,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:04,208 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:04,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:04,215 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:04,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:05,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:05,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:09,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:09,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:09,855 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:09,861 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:10,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:10,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:11,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:13,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:13,099 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:13,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:13,106 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:13,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:14,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:14,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:15,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:16,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:17,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:17,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:17,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:17,315 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:17,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:19,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:21,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:21,105 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:21,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:21,113 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:21,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:22,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:23,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:24,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:44:26,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:26,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:26,774 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:26,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:27,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:27,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:27,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:29,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:29,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:30,760 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:30,760 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:30,761 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:30,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:31,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:31,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:32,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:34,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:34,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:34,471 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:34,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:34,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:35,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:38,478 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:40,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:40,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:40,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:40,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:40,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:41,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:44,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:44,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:44,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:44,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:44,836 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:44,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:44,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:45,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:45,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:49,530 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:49,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:49,531 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:49,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:49,538 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:49,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:51,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:54,367 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:44:54,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:54,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:54,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:54,475 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:54,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:55,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:55,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:58,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:58,522 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:58,522 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:58,529 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:58,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:59,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:59,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:59,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:59,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:01,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:02,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:02,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:02,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:02,743 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:03,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:03,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:06,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:07,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:07,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:07,283 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:07,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:07,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:09,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:11,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:11,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:11,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:11,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:11,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:12,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:13,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:14,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:14,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:16,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:16,419 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:16,419 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:16,425 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:16,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:17,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:17,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:20,182 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:20,183 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:20,183 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:20,188 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:20,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:21,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:23,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:24,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:45:24,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:24,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:24,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:24,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:25,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:25,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:28,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:28,798 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:28,798 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:28,799 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:28,804 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:29,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:29,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:29,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:29,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:33,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:33,344 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:33,344 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:33,350 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:33,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:34,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:35,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:37,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:37,560 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:37,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:37,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:37,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:39,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:39,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:41,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:41,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:41,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:41,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:41,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:43,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:44,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:45,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:45,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:45,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:45,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:45,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:45,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:47,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:50,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:54,371 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:45:54,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:54,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:54,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:54,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:54,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:55,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:56,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:59,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:59,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:59,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:59,364 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:59,364 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:59,369 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:59,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:01,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:02,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:06,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:06,051 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:06,051 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:06,056 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:06,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:07,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:08,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:09,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:09,775 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:09,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:09,780 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:10,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:11,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:13,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:13,958 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:13,959 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:13,959 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:13,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:14,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:14,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:14,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:15,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:19,274 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:22,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:22,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:22,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:22,998 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:23,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:23,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:24,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:46:24,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:27,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:27,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:27,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:27,811 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:28,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:29,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:29,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:29,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:30,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:32,799 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:32,800 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:32,800 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:32,805 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:33,779 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:33,779 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:35,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:37,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:37,159 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:37,159 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:37,164 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:37,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:39,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:41,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:43,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:43,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:43,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:43,340 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:43,789 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:44,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:44,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:45,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:46,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:47,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:47,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:47,257 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:47,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:47,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:49,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:52,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:52,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:52,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:52,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:52,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:53,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:53,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:54,381 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:46:56,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:56,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:56,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:56,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:56,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:57,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:57,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:59,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:01,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:01,087 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:01,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:01,092 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:01,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:01,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:03,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:04,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:04,950 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:04,951 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:04,956 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:05,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:05,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:08,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:08,694 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:08,695 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:08,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:08,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:08,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:09,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:13,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:13,423 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:13,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:13,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:13,824 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:14,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:14,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:14,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:15,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:17,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:17,807 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:17,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:17,813 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:17,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:19,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:19,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:21,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:21,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:21,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:21,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:22,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:23,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:24,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:47:25,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:29,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:29,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:30,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:30,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:30,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:30,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:30,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:30,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:31,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:35,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:35,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:35,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:35,123 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:35,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:35,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:36,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:38,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:38,658 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:38,658 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:38,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:38,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:39,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:41,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:42,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:42,419 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:42,419 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:42,424 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:42,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:43,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:44,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:44,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:46,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:46,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:46,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:46,115 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:46,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:47,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:47,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:50,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:50,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:50,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:50,550 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:50,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:51,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:52,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:54,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:54,278 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:54,278 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:54,287 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:54,387 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:47:54,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:55,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:58,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:58,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:58,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:58,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:58,372 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:58,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:59,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:59,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:02,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:02,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:02,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:02,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:02,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:03,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:03,883 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:08,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:10,839 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:10,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:10,840 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:10,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:10,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:11,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:13,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:14,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:14,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:17,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:17,067 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:17,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:17,072 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:17,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:17,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:19,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:24,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:24,383 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:48:26,136 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:26,137 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:26,137 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:26,142 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:26,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:27,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:29,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:29,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:30,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:30,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:30,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:30,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:30,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:31,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:34,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:38,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:38,474 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:38,474 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:38,479 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:38,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:39,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:39,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:42,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:42,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:42,658 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:42,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:42,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:43,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:44,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:44,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:45,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:46,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:46,281 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:46,281 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:46,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:46,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:47,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:50,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:50,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:50,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:50,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:50,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:50,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:51,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:54,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:54,301 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:54,301 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:54,306 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:54,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:48:54,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:55,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:55,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:58,538 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:58,539 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:58,540 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:58,544 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:58,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:59,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:59,942 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:01,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:06,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:06,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:06,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:06,676 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:06,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:06,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:07,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:10,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:10,403 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:10,404 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:10,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:10,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:11,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:11,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:13,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:13,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:13,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:13,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:13,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:14,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:14,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:15,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:17,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:18,908 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:18,909 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:18,909 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:18,914 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:18,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:19,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:22,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:23,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:23,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:23,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:23,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:23,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:23,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:24,387 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:49:28,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:28,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:28,600 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:28,600 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:28,605 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:28,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:29,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:29,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:29,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:29,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:32,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:32,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:32,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:32,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:32,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:33,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:34,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:35,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:35,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:35,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:35,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:36,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:37,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:39,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:41,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:41,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:41,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:41,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:41,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:43,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:44,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:44,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:44,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:45,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:45,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:45,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:45,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:45,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:45,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:47,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:49,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:49,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:49,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:49,247 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:49,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:49,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:51,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:53,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:53,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:53,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:53,247 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:53,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:53,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:54,389 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:49:56,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:59,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:59,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:59,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:01,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:01,468 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:01,468 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:01,468 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:01,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:02,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:04,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:05,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:05,631 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:05,631 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:05,636 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:06,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:06,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:08,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:09,488 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:09,488 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:09,489 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:09,494 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:10,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:12,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:12,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:14,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:14,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:14,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:14,500 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:14,500 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:14,505 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:15,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:16,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:17,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:18,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:18,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:18,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:18,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:19,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:20,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:23,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:24,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:50:24,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:24,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:24,865 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:24,870 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:25,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:26,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:28,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:28,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:28,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:28,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:28,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:29,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:29,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:29,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:29,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:30,037 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:34,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:34,047 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:34,047 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:34,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:34,052 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:35,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:36,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:38,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:38,381 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:38,382 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:38,386 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:39,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:39,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:40,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:43,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:43,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:43,923 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:43,928 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:44,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:44,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:44,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:44,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:45,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:46,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:48,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:48,165 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:48,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:48,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:49,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:50,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:51,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:52,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:52,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:52,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:52,886 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:53,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:54,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:54,393 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:50:56,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:56,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:56,836 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:56,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:56,841 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:57,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:58,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:59,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:59,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:59,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:00,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:00,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:00,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:00,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:01,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:01,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:02,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:04,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:04,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:04,975 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:04,980 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:05,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:06,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:06,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:09,460 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:09,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:09,461 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:09,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:10,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:10,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:12,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:14,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:14,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:14,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:15,427 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:15,428 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:15,428 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:15,433 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:16,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:16,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:17,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:19,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:19,819 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:19,819 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:19,824 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:20,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:22,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:22,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:24,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:24,136 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:24,136 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:24,142 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:24,398 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:51:25,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:26,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:28,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:28,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:28,643 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:28,643 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:28,648 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:29,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:29,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:29,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:29,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:30,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:32,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:32,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:32,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:32,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:33,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:33,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:34,106 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:38,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:41,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:41,070 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:41,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:41,075 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:41,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:42,113 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:44,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:44,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:44,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:45,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:45,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:45,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:45,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:46,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:48,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:48,975 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:48,975 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:48,980 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:49,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:49,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:50,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:54,200 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:54,201 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:54,201 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:54,206 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:54,397 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:51:55,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:55,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:56,128 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:58,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:58,534 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:58,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:58,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:59,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:59,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:59,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:59,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:00,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:01,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:02,179 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:02,179 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:02,180 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:02,185 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:03,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:04,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:06,136 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:06,137 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:06,137 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:06,142 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:07,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:07,142 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:08,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:10,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:10,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:10,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:10,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:11,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:12,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:12,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:14,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:14,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:14,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:14,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:14,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:14,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:14,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:14,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:16,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:18,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:20,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:20,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:20,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:20,249 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:21,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:22,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:24,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:24,402 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:52:26,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:26,300 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:26,300 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:26,305 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:27,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:28,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:29,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:29,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:29,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:29,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:30,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:30,850 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:30,850 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:30,855 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:31,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:32,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:34,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:36,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:36,710 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:36,710 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:36,715 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:37,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:38,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:40,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:41,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:41,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:41,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:41,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:42,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:42,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:44,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:44,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:44,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:46,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:46,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:46,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:46,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:46,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:47,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:48,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:50,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:50,707 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:50,707 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:50,712 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:51,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:51,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:52,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:54,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:52:54,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:54,718 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:54,718 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:54,723 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:55,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:56,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:56,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:58,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:58,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:58,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:58,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:59,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:59,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:59,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:00,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:02,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:03,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:03,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:03,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:03,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:04,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:04,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:07,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:08,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:08,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:08,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:08,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:08,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:10,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:12,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:12,786 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:12,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:12,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:12,791 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:13,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:14,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:14,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:14,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:17,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:17,002 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:17,002 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:17,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:17,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:18,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:18,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:23,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:23,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:23,939 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:23,940 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:23,945 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:24,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:24,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:53:26,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:28,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:29,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:29,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:32,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:32,845 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:32,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:32,850 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:33,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:33,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:34,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:38,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:41,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:41,117 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:41,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:41,122 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:41,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:42,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:44,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:44,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:44,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:49,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:49,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:49,541 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:49,541 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:49,546 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:50,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:50,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:54,409 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:53:54,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:55,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:55,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:55,855 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:55,860 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:56,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:58,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:59,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:59,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:00,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:04,182 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:04,183 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:04,183 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:04,188 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:04,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:06,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:06,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:08,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:08,223 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:08,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:08,228 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:08,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:10,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:11,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:12,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:12,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:12,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:12,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:13,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:14,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:14,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:14,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:16,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:17,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:17,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:17,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:17,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:17,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:18,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:22,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:22,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:22,775 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:22,776 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:22,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:23,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:24,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:24,408 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:54:27,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:27,339 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:27,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:27,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:27,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:28,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:28,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:29,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:29,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:33,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:33,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:33,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:33,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:33,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:34,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:34,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:37,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:37,601 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:37,602 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:37,606 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:38,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:38,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:38,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:42,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:42,383 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:42,383 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:42,389 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:43,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:44,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:44,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:44,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:46,496 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:46,497 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:46,497 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:46,502 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:47,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:48,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:49,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:51,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:51,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:51,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:51,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:52,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:52,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:54,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:54:54,820 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:54,821 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:54,821 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:54,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:54,826 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:55,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:56,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:59,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:59,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:59,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:59,570 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:59,570 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:59,575 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:00,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:00,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:00,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:05,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:07,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:07,948 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:07,948 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:07,953 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:08,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:10,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:10,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:12,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:12,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:12,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:12,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:13,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:14,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:14,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:14,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:16,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:16,093 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:16,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:16,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:16,099 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:16,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:18,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:21,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:21,735 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:21,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:21,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:21,741 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:22,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:22,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:24,412 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:55:26,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:26,158 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:26,159 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:26,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:26,163 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:26,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:28,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:29,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:29,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:31,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:35,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:35,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:35,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:35,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:36,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:36,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:36,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:40,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:40,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:40,039 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:40,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:40,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:42,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:42,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:43,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:43,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:43,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:43,879 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:44,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:44,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:44,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:46,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:47,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:48,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:48,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:48,432 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:48,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:49,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:50,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:52,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:52,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:52,454 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:52,454 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:52,459 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:53,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:54,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:54,414 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:55:56,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:56,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:56,360 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:56,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:56,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:58,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:58,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:59,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:59,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:00,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:00,157 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:00,158 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:00,162 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:00,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:02,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:04,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:05,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:05,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:05,252 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:05,257 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:05,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:06,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:09,258 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:11,720 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:11,721 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:11,721 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:11,726 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:12,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:12,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:14,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:14,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:14,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:14,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:16,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:16,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:16,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:16,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:17,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:18,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:19,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:21,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:21,272 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:21,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:21,278 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:21,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:22,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:24,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:56:25,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:27,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:27,206 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:27,207 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:27,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:27,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:28,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:29,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:29,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:29,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:31,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:31,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:31,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:31,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:31,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:32,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:34,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:35,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:35,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:35,831 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:35,836 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:36,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:36,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:36,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:41,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:41,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:41,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:41,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:41,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:42,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:42,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:44,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:44,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:44,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:45,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:45,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:45,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:45,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:45,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:46,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:47,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:48,806 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:48,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:48,807 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:48,812 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:49,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:50,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:52,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:53,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:53,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:53,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:53,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:54,418 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:56:54,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:54,431 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:58,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:59,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:59,015 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:59,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:59,021 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:59,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:59,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:59,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:59,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:00,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:03,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:03,802 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:03,802 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:03,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:03,807 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:04,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:04,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:08,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:08,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:08,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:08,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:08,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:09,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:10,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:12,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:12,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:12,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:12,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:12,447 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:14,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:14,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:14,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:14,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:14,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:15,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:15,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:15,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:15,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:16,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:16,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:19,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:19,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:19,572 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:19,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:19,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:20,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:20,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:23,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:23,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:23,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:23,594 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:24,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:57:24,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:24,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:25,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:28,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:28,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:28,949 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:28,954 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:29,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:29,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:29,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:29,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:30,470 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:31,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:33,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:33,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:33,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:33,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:33,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:34,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:36,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:39,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:39,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:39,614 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:39,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:40,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:40,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:41,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:44,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:44,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:45,472 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:45,473 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:45,473 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:45,478 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:45,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:46,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:47,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:51,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:51,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:51,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:51,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:52,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:52,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:52,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:54,423 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:57:57,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:57,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:57,641 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:57,646 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:58,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:58,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:58,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:59,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:59,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:59,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:02,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:02,859 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:02,859 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:02,865 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:03,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:03,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:04,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:07,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:07,352 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:07,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:07,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:07,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:08,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:09,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:11,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:11,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:11,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:11,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:12,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:12,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:14,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:14,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:14,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:15,267 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:16,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:16,834 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:16,834 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:16,839 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:17,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:18,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:20,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:21,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:21,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:21,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:21,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:21,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:22,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:24,429 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:58:26,430 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:27,746 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:27,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:27,747 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:27,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:28,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:28,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:29,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:29,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:29,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:32,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:35,907 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:35,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:35,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:35,913 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:36,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:36,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:37,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:40,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:40,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:40,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:40,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:40,539 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:42,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:43,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:44,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:44,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:44,259 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:44,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:44,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:44,292 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:44,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:46,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:48,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:48,043 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:48,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:48,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:48,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:48,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:50,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:53,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:53,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:53,299 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:53,300 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:53,304 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:53,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:54,427 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:58:54,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:57,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:57,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:57,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:57,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:57,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:58,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:59,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:59,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:59,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:59,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:01,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:01,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:01,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:01,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:01,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:02,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:05,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:06,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:06,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:06,014 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:06,020 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:06,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:08,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:10,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:10,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:10,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:10,863 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:10,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:11,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:12,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:14,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:14,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:14,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:15,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:15,186 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:15,187 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:15,192 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:15,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:16,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:16,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:19,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:19,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:19,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:19,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:19,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:20,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:21,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:23,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:23,221 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:23,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:23,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:23,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:24,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:59:24,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:27,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:28,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:28,107 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:28,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:28,112 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:28,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:29,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:29,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:29,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:30,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:32,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:32,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:32,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:32,190 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:32,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:33,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:34,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:37,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:37,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:37,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:37,106 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:37,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:38,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:39,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:44,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:44,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:45,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:45,489 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:45,489 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:45,494 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:45,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:46,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:49,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:53,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:53,838 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:53,838 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:53,843 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:54,431 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:59:54,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:54,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:55,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:57,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:57,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:57,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:57,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:57,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:58,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:59,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:59,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:59,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:01,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:03,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:03,028 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:03,028 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:03,033 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:03,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:04,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:07,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:07,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:07,485 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:07,485 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:07,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:07,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:08,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:11,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:11,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:11,169 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:11,174 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:11,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:12,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:12,635 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:14,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:14,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:14,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:15,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:15,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:15,288 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:15,293 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:15,637 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:16,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:17,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:19,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:19,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:19,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:19,043 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:19,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:20,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:23,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:23,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:23,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:23,647 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:23,652 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:24,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:00:24,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:24,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:28,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:29,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:29,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:29,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:34,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:34,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:34,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:34,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:34,258 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:34,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:36,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:39,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:42,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:42,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:42,283 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:42,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:42,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:44,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:44,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:44,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:44,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:49,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:50,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:50,519 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:50,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:50,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:50,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:52,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:54,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:54,062 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:54,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:54,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:54,438 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:00:54,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:54,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:54,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:59,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:59,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:59,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:59,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:59,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:59,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:59,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:59,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:00,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:03,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:03,823 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:03,823 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:03,828 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:04,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:04,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:04,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:07,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:07,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:07,405 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:07,410 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:07,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:08,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:10,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:11,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:11,218 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:11,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:11,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:11,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:12,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:14,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:14,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:14,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:14,820 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:14,821 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:14,821 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:14,826 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:15,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:15,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:16,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:18,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:18,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:18,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:18,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:18,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:20,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:21,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:23,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:23,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:23,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:23,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:23,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:24,437 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:01:24,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:27,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:29,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:29,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:29,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:29,538 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:29,539 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:29,539 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:29,544 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:29,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:30,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:32,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:36,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:36,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:36,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:36,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:36,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:38,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:38,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:40,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:40,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:40,031 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:40,036 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:40,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:40,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:44,016 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:44,017 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:44,017 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:44,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:44,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:44,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:44,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:44,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:44,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:44,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:49,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:50,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:50,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:50,148 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:50,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:50,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:50,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:53,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:53,801 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:53,802 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:53,807 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:54,442 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:01:54,444 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:54,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:54,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:58,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:58,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:58,217 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:58,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:58,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:59,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:59,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:00,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:00,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:02,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:02,268 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:02,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:02,274 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:02,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:04,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:05,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:07,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:07,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:07,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:07,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:07,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:08,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:11,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:12,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:12,892 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:12,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:12,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:13,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:14,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:14,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:14,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:14,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:16,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:17,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:17,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:17,138 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:17,143 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:17,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:18,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:21,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:21,136 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:21,136 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:21,142 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:21,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:22,142 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:22,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:24,441 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:02:26,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:26,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:26,719 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:26,724 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:26,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:27,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:28,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:29,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:29,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:29,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:33,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:35,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:35,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:35,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:35,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:35,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:36,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:39,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:43,452 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:43,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:43,453 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:43,459 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:43,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:44,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:44,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:44,284 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:44,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:47,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:47,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:47,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:47,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:47,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:48,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:49,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:51,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:51,533 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:51,533 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:51,538 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:51,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:52,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:54,443 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:02:55,210 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:55,211 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:55,211 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:55,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:55,216 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:55,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:56,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:59,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:59,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:59,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:59,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:59,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:59,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:00,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:00,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:05,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:08,700 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:08,700 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:08,701 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:08,706 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:08,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:10,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:11,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:12,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:12,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:12,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:12,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:12,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:12,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:14,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:14,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:14,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:16,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:16,181 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:16,182 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:16,187 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:16,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:16,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:17,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:20,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:20,248 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:20,249 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:20,254 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:20,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:20,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:22,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:24,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:24,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:24,299 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:24,304 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:24,446 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:03:24,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:24,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:27,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:28,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:28,458 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:28,459 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:28,464 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:28,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:29,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:29,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:29,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:30,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:32,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:32,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:32,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:32,230 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:32,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:32,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:33,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:38,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:38,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:38,270 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:38,271 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:38,275 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:38,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:38,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:43,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:44,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:44,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:46,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:46,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:46,686 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:46,691 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:46,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:48,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:48,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:50,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:50,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:50,421 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:50,427 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:50,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:52,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:54,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:54,449 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:03:55,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:55,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:55,957 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:55,962 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:56,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:56,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:59,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:59,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:59,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:59,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:59,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:59,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:59,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:59,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:00,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:04,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:04,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:04,508 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:04,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:04,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:05,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:06,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:10,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:13,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:13,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:13,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:13,312 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:13,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:14,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:14,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:14,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:14,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:16,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:17,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:17,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:17,626 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:17,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:17,883 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:18,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:21,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:22,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:22,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:22,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:22,584 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:22,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:24,451 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:04:24,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:26,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:26,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:26,295 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:26,300 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:26,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:26,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:27,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:29,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:29,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:29,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:30,708 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:30,709 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:30,709 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:30,715 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:30,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:32,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:32,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:35,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:35,625 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:35,626 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:35,631 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:35,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:36,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:38,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:40,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:40,532 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:40,533 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:40,538 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:40,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:42,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:44,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:44,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:44,284 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:45,496 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:45,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:45,497 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:45,502 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:45,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:46,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:49,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:49,364 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:49,365 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:49,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:49,370 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:49,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:50,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:54,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:54,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:04:54,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:54,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:54,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:54,921 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:54,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:56,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:58,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:58,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:58,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:58,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:58,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:59,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:59,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:59,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:00,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:00,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:02,648 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:02,649 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:02,649 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:02,654 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:02,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:04,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:05,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:07,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:07,565 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:07,565 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:07,570 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:07,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:08,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:11,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:11,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:11,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:11,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:11,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:11,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:12,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:14,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:14,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:14,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:16,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:16,341 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:16,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:16,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:16,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:16,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:17,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:21,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:21,196 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:21,196 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:21,207 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:21,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:22,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:23,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:24,464 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:05:25,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:25,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:25,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:25,637 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:25,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:26,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:28,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:29,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:29,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:29,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:33,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:33,930 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:33,930 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:33,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:33,940 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:33,979 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:34,987 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:38,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:39,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:39,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:39,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:39,326 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:39,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:40,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:43,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:43,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:43,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:43,123 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:43,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:44,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:44,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:44,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:44,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:45,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:47,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:47,087 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:47,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:47,093 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:48,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:49,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:49,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:51,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:51,077 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:51,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:51,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:52,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:53,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:54,457 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:05:54,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:55,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:55,455 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:55,456 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:55,468 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:56,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:57,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:59,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:59,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:00,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:01,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:01,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:01,283 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:01,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:02,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:03,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:06,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:06,426 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:06,426 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:06,427 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:06,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:07,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:07,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:10,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:10,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:10,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:10,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:11,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:11,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:13,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:14,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:14,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:14,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:15,012 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:15,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:15,013 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:15,022 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:15,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:17,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:17,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:18,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:18,641 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:18,641 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:18,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:19,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:21,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:22,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:22,162 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:22,163 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:22,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:22,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:23,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:23,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:24,469 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:06:27,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:27,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:27,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:27,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:27,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:28,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:29,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:29,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:29,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:29,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:31,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:31,513 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:31,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:31,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:32,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:32,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:33,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:37,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:39,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:39,449 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:39,450 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:39,455 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:40,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:41,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:43,456 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:43,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:43,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:43,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:43,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:44,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:44,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:44,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:44,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:45,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:48,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:48,033 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:48,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:48,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:48,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:49,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:49,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:52,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:52,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:52,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:52,987 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:53,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:54,461 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:06:54,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:55,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:58,895 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:58,896 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:58,897 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:58,906 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:59,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:59,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:59,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:00,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:01,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:03,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:03,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:03,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:03,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:04,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:05,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:05,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:07,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:07,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:07,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:07,326 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:08,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:09,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:11,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:11,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:11,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:11,885 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:11,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:12,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:13,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:14,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:14,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:14,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:17,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:20,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:20,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:20,246 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:20,254 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:21,173 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:21,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:22,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:24,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:07:24,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:24,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:24,835 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:24,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:25,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:27,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:27,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:29,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:29,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:29,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:30,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:30,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:30,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:30,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:31,187 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:31,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:33,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:35,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:35,570 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:35,570 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:35,576 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:36,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:37,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:38,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:41,871 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:41,872 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:41,872 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:41,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:42,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:43,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:43,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:44,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:44,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:44,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:49,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:50,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:50,325 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:50,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:50,331 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:51,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:51,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:54,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:54,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:54,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:54,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:54,212 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:54,465 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:07:54,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:55,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:59,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:59,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:59,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:59,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:59,759 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:59,759 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:59,765 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:00,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:01,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:03,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:03,718 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:03,719 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:03,726 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:04,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:05,225 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:05,727 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:07,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:07,474 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:07,475 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:07,480 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:08,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:09,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:11,481 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:12,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:12,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:12,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:12,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:13,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:13,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:14,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:14,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:14,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:16,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:16,596 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:16,597 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:16,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:16,602 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:17,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:17,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:20,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:20,917 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:20,917 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:20,922 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:21,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:21,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:23,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:24,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:08:27,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:27,430 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:27,430 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:27,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:27,436 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:28,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:29,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:29,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:29,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:29,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:30,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:30,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:30,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:30,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:31,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:32,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:33,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:34,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:34,584 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:34,584 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:34,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:35,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:35,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:38,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:38,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:38,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:38,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:38,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:39,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:41,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:43,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:43,839 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:43,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:43,840 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:43,845 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:44,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:44,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:44,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:44,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:45,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:49,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:49,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:49,495 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:49,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:49,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:50,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:51,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:54,473 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:08:54,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:55,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:55,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:55,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:55,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:56,280 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:57,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:59,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:59,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:00,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:01,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:01,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:01,039 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:01,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:01,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:03,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:06,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:07,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:07,768 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:07,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:07,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:08,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:09,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:11,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:13,139 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:13,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:13,140 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:13,145 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:13,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:14,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:14,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:14,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:15,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:17,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:17,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:17,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:17,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:17,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:17,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:19,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:20,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:20,950 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:20,951 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:20,956 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:21,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:22,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:23,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:24,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:09:25,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:25,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:25,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:25,364 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:26,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:27,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:28,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:29,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:29,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:29,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:31,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:31,405 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:31,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:31,411 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:32,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:33,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:33,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:38,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:38,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:38,815 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:38,815 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:38,821 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:39,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:41,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:43,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:43,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:43,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:43,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:43,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:44,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:44,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:44,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:44,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:45,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:48,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:48,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:48,040 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:48,045 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:48,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:49,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:49,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:54,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:54,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:54,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:54,299 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:54,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:54,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:54,474 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:09:55,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:58,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:58,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:58,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:58,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:59,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:59,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:59,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:59,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:04,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:04,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:04,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:04,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:04,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:05,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:05,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:08,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:08,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:08,688 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:08,693 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:09,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:09,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:09,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:12,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:12,658 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:12,658 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:12,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:13,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:13,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:14,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:14,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:14,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:15,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:16,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:16,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:16,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:16,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:17,361 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:19,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:20,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:22,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:22,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:22,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:22,317 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:22,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:23,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:24,477 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:10:26,478 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:29,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:29,033 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:29,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:29,039 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:29,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:29,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:29,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:29,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:31,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:32,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:35,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:35,206 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:35,206 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:35,211 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:35,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:37,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:38,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:41,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:41,650 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:41,651 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:41,656 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:42,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:43,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:43,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:44,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:44,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:44,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:46,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:46,015 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:46,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:46,020 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:46,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:47,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:49,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:49,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:49,802 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:49,803 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:49,807 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:50,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:51,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:54,479 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:10:54,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:55,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:55,051 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:55,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:55,057 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:55,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:57,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:58,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:58,523 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:58,523 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:58,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:59,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:59,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:59,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:59,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:00,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:02,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:02,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:02,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:02,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:02,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:03,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:05,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:07,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:07,406 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:07,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:07,411 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:07,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:09,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:10,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:12,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:12,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:12,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:12,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:12,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:13,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:14,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:14,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:16,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:16,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:16,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:16,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:16,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:17,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:17,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:21,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:21,129 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:21,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:21,134 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:21,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:22,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:23,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:24,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:11:25,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:25,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:25,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:25,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:25,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:27,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:28,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:29,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:29,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:29,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:31,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:31,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:31,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:31,779 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:32,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:33,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:33,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:38,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:40,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:40,181 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:40,182 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:40,187 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:40,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:41,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:44,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:44,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:44,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:44,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:44,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:44,340 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:44,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:44,345 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:44,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:45,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:47,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:47,837 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:47,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:47,842 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:48,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:49,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:49,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:51,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:51,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:51,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:51,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:52,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:53,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:54,483 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:11:55,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:57,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:57,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:57,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:57,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:57,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:59,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:59,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:59,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:59,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:01,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:02,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:02,470 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:02,470 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:02,475 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:03,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:03,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:06,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:07,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:07,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:07,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:07,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:07,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:09,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:12,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:12,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:12,177 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:12,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:12,182 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:12,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:13,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:14,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:14,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:16,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:16,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:16,831 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:16,836 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:17,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:17,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:17,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:20,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:20,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:20,366 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:20,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:20,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:21,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:23,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:24,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:24,382 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:24,382 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:24,387 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:24,489 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:12:24,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:25,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:28,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:29,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:29,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:29,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:29,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:29,683 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:29,683 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:29,688 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:30,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:31,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:33,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:33,313 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:33,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:33,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:33,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:34,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:35,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:36,735 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:36,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:36,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:36,741 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:37,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:37,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:39,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:41,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:41,352 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:41,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:41,358 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:41,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:43,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:44,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:44,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:44,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:45,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:45,334 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:45,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:45,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:45,339 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:45,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:47,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:49,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:49,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:49,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:49,094 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:49,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:51,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:51,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:52,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:52,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:52,831 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:52,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:53,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:53,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:54,487 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:12:56,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:57,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:57,930 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:57,930 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:57,935 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:58,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:59,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:59,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:59,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:02,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:02,867 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:02,868 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:02,868 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:02,873 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:03,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:03,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:07,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:08,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:08,153 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:08,154 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:08,159 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:08,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:09,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:13,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:14,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:14,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:15,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:15,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:15,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:15,292 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:15,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:17,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:18,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:19,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:19,286 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:19,286 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:19,292 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:19,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:21,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:24,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:24,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:24,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:24,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:24,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:24,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:13:24,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:25,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:27,920 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:27,921 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:27,921 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:27,926 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:28,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:29,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:29,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:29,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:29,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:29,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:31,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:31,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:31,659 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:31,664 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:32,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:33,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:34,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:35,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:35,982 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:35,982 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:35,987 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:36,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:37,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:39,966 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:39,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:39,967 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:39,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:39,972 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:40,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:41,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:44,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:44,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:44,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:44,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:44,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:44,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:44,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:44,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:45,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:45,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:48,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:48,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:48,222 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:48,227 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:48,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:49,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:51,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:51,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:51,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:51,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:51,999 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:52,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:53,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:54,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:13:55,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:55,958 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:55,958 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:55,963 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:56,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:56,963 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:57,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:59,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:59,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:00,872 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:00,873 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:00,873 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:00,878 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:01,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:01,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:02,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:06,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:06,585 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:06,585 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:06,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:06,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:07,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:08,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:11,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:11,524 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:11,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:11,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:11,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:13,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:14,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:14,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:14,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:14,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:14,812 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:14,813 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:14,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:15,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:15,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:18,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:18,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:18,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:18,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:19,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:19,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:19,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:23,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:23,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:23,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:23,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:23,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:23,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:24,497 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:14:25,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:27,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:27,076 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:27,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:27,081 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:27,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:29,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:29,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:29,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:29,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:31,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:31,060 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:31,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:31,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:31,064 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:31,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:33,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:36,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:37,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:37,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:37,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:37,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:38,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:39,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:41,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:43,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:43,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:43,136 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:43,141 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:43,637 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:44,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:44,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:44,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:45,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:47,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:47,098 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:47,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:47,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:47,103 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:47,642 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:49,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:52,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:54,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:14:55,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:55,573 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:55,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:55,578 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:55,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:57,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:57,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:59,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:59,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:59,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:59,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:59,678 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:59,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:00,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:01,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:02,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:03,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:03,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:03,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:03,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:04,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:05,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:07,910 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:08,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:08,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:08,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:08,744 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:09,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:09,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:13,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:13,228 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:13,229 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:13,230 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:13,234 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:13,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:14,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:14,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:14,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:15,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:16,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:16,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:16,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:16,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:17,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:17,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:19,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:22,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:22,383 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:22,383 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:22,388 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:22,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:23,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:24,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:24,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:15:27,856 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:27,857 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:27,857 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:27,862 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:28,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:29,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:29,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:29,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:29,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:30,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:32,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:32,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:32,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:32,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:32,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:33,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:36,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:37,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:37,103 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:37,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:37,108 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:37,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:37,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:41,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:44,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:44,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:44,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:45,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:45,269 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:45,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:45,275 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:45,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:46,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:47,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:50,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:50,478 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:50,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:50,484 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:50,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:51,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:51,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:54,506 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:15:55,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:55,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:55,629 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:55,633 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:55,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:56,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:57,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:59,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:59,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:01,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:01,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:01,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:01,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:01,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:01,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:03,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:05,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:05,445 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:05,446 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:05,451 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:05,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:07,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:07,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:10,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:10,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:10,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:10,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:10,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:11,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:13,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:14,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:14,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:14,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:14,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:14,391 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:14,391 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:14,396 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:14,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:15,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:18,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:21,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:21,109 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:21,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:21,114 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:21,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:21,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:24,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:24,502 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:16:26,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:26,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:26,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:26,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:26,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:27,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:29,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:29,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:29,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:29,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:30,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:30,299 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:30,299 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:30,304 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:30,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:31,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:34,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:34,011 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:34,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:34,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:34,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:35,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:35,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:40,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:40,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:40,050 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:40,050 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:40,055 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:40,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:41,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:44,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:44,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:44,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:44,647 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:44,648 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:44,648 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:44,653 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:44,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:45,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:45,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:48,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:48,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:48,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:48,697 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:48,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:49,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:50,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:54,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:16:54,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:54,584 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:54,584 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:54,589 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:54,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:55,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:56,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:58,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:58,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:58,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:58,895 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:59,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:59,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:59,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:59,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:59,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:02,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:04,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:04,345 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:04,345 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:04,350 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:04,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:05,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:07,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:08,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:08,691 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:08,691 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:08,696 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:08,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:09,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:12,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:14,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:14,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:17,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:17,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:17,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:17,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:17,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:17,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:18,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:20,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:20,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:20,954 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:20,959 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:21,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:21,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:23,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:24,507 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:17:27,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:27,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:27,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:27,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:28,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:29,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:29,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:29,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:29,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:29,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:31,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:31,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:31,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:31,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:31,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:33,810 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:34,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:35,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:35,850 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:35,850 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:35,855 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:36,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:37,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:39,856 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:40,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:40,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:40,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:40,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:40,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:41,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:44,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:44,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:44,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:44,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:44,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:44,481 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:44,486 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:44,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:45,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:45,824 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:50,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:50,827 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:50,827 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:50,828 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:50,833 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:51,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:51,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:54,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:17:54,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:54,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:54,771 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:54,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:54,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:55,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:55,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:58,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:58,723 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:58,723 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:58,728 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:58,836 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:59,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:59,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:59,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:59,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:01,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:02,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:02,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:02,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:02,760 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:02,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:03,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:06,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:07,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:07,093 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:07,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:07,098 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:07,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:07,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:12,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:13,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:13,737 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:13,738 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:13,743 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:13,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:14,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:14,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:15,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:17,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:17,064 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:17,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:17,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:17,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:17,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:17,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:21,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:21,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:21,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:21,557 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:21,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:22,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:23,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:24,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:18:28,398 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:28,399 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:28,399 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:28,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:28,404 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:28,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:29,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:29,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:29,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:29,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:32,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:32,352 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:32,353 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:32,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:32,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:33,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:34,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:37,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:37,219 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:37,219 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:37,224 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:37,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:37,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:40,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:41,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:41,442 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:41,442 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:41,447 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:41,880 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:43,883 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:44,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:44,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:44,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:45,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:47,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:47,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:47,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:47,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:47,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:49,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:50,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:52,831 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:52,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:52,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:52,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:52,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:53,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:54,513 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:18:55,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:59,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:59,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:59,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:00,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:00,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:00,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:00,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:00,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:00,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:01,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:04,423 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:04,424 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:04,424 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:04,429 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:04,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:05,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:06,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:08,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:08,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:08,886 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:08,891 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:08,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:09,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:11,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:13,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:13,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:13,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:13,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:13,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:13,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:14,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:14,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:16,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:16,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:16,879 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:16,884 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:16,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:16,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:17,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:21,811 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:21,812 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:21,812 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:21,817 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:21,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:22,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:23,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:24,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:19:27,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:27,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:27,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:27,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:27,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:28,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:29,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:29,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:29,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:29,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:30,976 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:30,977 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:30,977 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:30,982 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:31,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:31,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:34,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:35,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:35,846 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:35,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:35,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:35,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:37,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:39,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:39,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:39,909 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:39,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:39,915 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:39,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:41,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:43,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:43,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:43,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:43,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:43,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:43,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:44,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:44,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:44,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:45,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:48,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:48,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:48,599 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:48,604 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:48,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:49,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:50,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:54,517 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:19:55,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:55,325 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:55,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:55,331 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:55,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:55,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:56,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:58,857 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:58,858 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:58,858 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:58,863 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:58,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:59,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:59,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:59,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:59,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:02,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:05,806 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:05,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:05,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:05,812 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:05,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:07,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:07,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:10,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:10,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:10,482 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:10,486 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:10,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:11,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:13,487 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:14,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:14,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:14,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:15,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:15,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:15,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:15,043 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:15,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:15,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:19,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:19,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:19,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:19,207 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:19,212 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:19,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:19,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:23,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:23,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:23,351 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:23,356 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:23,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:23,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:24,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:24,522 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:20:27,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:27,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:27,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:27,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:27,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:27,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:29,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:29,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:29,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:30,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:31,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:31,571 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:31,571 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:31,576 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:31,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:33,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:35,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:35,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:35,779 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:35,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:35,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:35,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:38,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:40,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:40,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:40,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:40,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:41,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:41,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:42,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:44,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:44,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:44,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:44,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:44,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:44,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:44,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:45,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:46,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:47,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:48,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:48,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:48,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:48,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:49,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:50,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:52,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:53,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:53,056 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:53,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:53,061 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:54,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:54,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:54,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:20:57,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:57,101 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:57,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:57,107 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:58,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:58,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:58,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:59,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:59,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:59,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:01,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:01,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:01,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:01,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:02,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:02,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:03,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:05,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:05,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:05,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:05,094 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:06,033 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:06,033 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:09,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:12,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:12,019 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:12,019 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:12,024 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:12,037 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:14,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:14,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:14,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:14,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:14,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:15,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:15,707 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:15,707 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:15,712 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:16,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:18,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:19,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:19,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:19,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:19,854 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:19,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:20,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:22,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:23,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:23,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:23,634 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:23,639 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:24,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:24,527 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:21:25,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:26,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:29,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:29,024 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:29,025 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:29,030 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:29,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:29,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:29,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:29,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:30,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:31,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:34,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:34,043 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:34,043 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:34,048 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:34,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:36,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:37,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:42,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:43,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:43,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:43,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:43,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:44,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:44,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:44,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:44,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:47,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:47,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:47,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:47,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:47,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:48,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:50,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:52,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:52,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:52,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:52,358 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:53,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:53,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:54,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:54,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:21:55,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:55,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:55,831 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:55,836 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:56,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:58,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:58,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:59,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:59,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:01,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:01,165 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:01,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:01,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:02,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:02,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:04,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:06,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:06,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:06,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:06,692 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:07,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:08,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:09,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:12,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:12,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:12,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:12,269 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:13,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:14,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:14,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:14,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:14,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:15,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:16,409 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:16,410 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:16,410 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:16,415 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:17,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:18,106 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:20,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:22,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:22,446 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:22,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:22,452 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:23,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:24,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:24,527 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:22:25,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:28,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:28,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:28,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:28,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:29,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:29,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:29,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:29,116 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:30,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:31,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:32,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:32,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:32,288 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:32,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:33,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:34,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:36,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:36,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:36,646 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:36,646 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:36,652 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:37,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:38,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:40,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:40,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:40,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:40,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:41,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:41,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:42,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:44,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:44,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:45,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:45,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:45,656 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:45,661 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:46,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:47,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:48,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:49,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:49,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:49,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:49,515 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:50,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:50,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:53,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:54,529 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:22:56,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:56,714 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:56,714 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:56,719 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:57,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:58,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:58,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:59,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:59,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:00,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:00,958 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:00,958 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:00,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:01,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:02,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:04,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:07,490 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:07,491 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:07,491 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:07,496 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:08,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:08,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:09,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:11,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:11,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:11,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:11,894 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:12,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:14,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:14,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:14,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:14,168 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:15,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:15,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:15,926 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:15,926 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:15,934 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:16,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:18,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:19,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:19,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:19,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:19,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:20,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:20,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:20,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:23,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:23,295 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:23,295 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:23,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:24,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:24,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:24,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:23:25,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:28,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:28,775 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:28,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:28,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:29,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:29,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:29,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:29,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:30,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:31,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:33,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:33,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:33,831 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:33,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:34,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:36,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:36,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:37,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:37,748 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:37,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:37,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:38,209 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:40,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:42,755 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:43,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:43,228 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:43,228 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:43,236 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:44,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:44,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:44,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:44,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:47,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:47,620 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:47,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:47,626 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:48,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:48,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:48,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:51,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:51,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:51,177 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:51,182 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:51,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:52,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:54,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:54,538 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:23:55,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:55,514 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:55,514 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:55,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:56,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:56,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:59,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:59,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:59,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:59,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:59,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:59,375 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:59,381 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:00,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:00,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:03,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:03,082 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:03,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:03,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:03,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:05,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:06,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:07,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:07,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:07,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:07,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:07,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:08,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:11,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:11,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:11,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:11,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:11,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:11,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:12,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:14,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:14,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:14,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:16,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:16,199 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:16,199 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:16,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:16,206 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:17,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:18,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:21,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:21,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:21,289 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:21,289 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:21,295 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:22,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:22,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:24,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:24:26,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:29,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:29,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:29,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:30,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:30,281 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:30,281 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:30,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:31,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:32,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:32,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:35,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:35,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:35,167 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:35,172 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:36,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:36,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:38,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:39,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:39,859 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:39,859 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:39,865 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:40,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:42,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:43,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:43,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:43,696 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:43,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:43,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:44,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:44,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:44,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:44,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:44,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:47,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:47,639 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:47,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:47,645 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:48,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:48,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:49,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:51,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:51,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:51,192 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:51,198 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:52,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:52,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:54,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:24:55,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:55,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:55,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:55,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:55,515 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:56,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:56,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:59,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:59,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:59,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:59,366 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:59,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:00,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:00,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:01,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:05,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:05,244 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:05,244 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:05,250 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:06,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:06,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:07,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:10,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:10,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:10,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:10,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:11,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:12,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:12,958 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:14,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:14,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:14,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:14,746 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:14,747 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:14,747 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:14,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:15,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:16,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:18,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:18,875 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:18,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:18,876 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:18,881 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:19,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:20,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:23,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:24,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:25:24,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:24,600 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:24,600 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:24,605 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:25,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:26,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:29,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:29,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:29,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:34,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:34,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:34,972 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:34,972 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:34,977 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:35,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:36,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:38,900 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:38,900 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:38,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:38,906 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:39,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:39,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:40,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:44,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:44,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:44,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:45,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:46,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:46,598 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:46,599 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:46,604 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:47,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:48,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:50,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:51,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:51,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:51,974 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:51,979 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:52,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:54,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:54,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:25:56,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:59,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:59,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:59,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:00,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:00,440 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:00,440 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:00,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:01,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:02,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:02,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:06,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:06,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:06,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:06,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:07,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:08,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:08,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:10,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:10,768 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:10,768 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:10,773 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:11,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:12,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:13,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:14,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:14,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:14,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:16,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:16,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:16,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:16,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:17,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:18,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:18,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:20,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:20,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:20,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:20,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:21,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:22,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:24,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:24,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:26:25,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:25,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:25,956 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:25,961 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:26,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:28,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:29,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:29,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:29,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:30,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:32,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:32,262 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:32,262 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:32,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:33,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:34,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:36,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:41,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:42,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:42,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:42,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:42,611 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:43,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:44,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:44,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:44,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:44,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:46,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:47,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:47,249 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:47,250 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:47,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:47,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:48,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:52,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:52,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:52,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:52,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:52,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:53,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:54,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:54,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:26:57,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:59,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:59,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:59,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:00,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:00,194 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:00,194 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:00,199 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:00,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:02,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:03,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:04,748 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:04,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:04,749 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:04,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:05,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:06,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:08,755 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:09,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:09,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:09,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:09,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:10,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:10,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:13,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:13,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:13,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:13,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:14,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:14,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:14,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:14,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:14,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:14,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:18,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:18,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:18,190 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:18,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:18,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:20,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:20,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:22,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:22,319 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:22,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:22,324 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:23,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:24,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:24,551 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:27:25,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:26,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:26,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:26,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:26,057 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:26,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:28,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:29,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:29,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:29,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:31,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:32,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:32,166 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:32,166 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:32,171 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:32,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:34,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:37,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:38,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:38,269 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:38,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:38,274 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:38,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:40,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:42,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:43,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:43,342 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:43,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:43,347 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:44,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:44,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:44,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:44,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:47,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:47,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:47,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:47,963 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:47,968 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:48,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:50,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:51,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:51,709 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:51,710 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:51,715 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:52,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:52,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:52,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:54,550 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:27:55,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:55,442 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:55,442 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:55,447 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:56,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:56,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:58,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:59,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:59,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:00,835 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:00,836 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:00,836 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:00,841 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:01,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:02,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:03,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:05,496 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:05,497 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:05,497 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:05,502 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:06,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:06,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:09,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:10,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:10,341 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:10,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:10,347 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:10,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:12,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:14,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:14,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:14,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:14,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:14,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:14,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:14,504 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:14,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:15,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:16,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:19,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:21,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:21,364 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:21,365 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:21,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:22,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:22,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:24,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:28:24,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:28,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:28,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:28,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:28,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:29,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:29,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:29,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:29,375 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:30,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:30,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:32,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:32,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:32,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:32,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:32,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:34,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:35,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:38,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:38,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:38,364 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:38,369 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:38,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:40,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:40,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:42,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:42,517 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:42,517 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:42,522 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:43,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:44,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:44,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:44,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:45,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:47,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:47,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:47,599 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:47,604 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:48,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:48,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:50,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:51,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:51,783 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:51,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:51,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:52,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:52,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:54,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:28:56,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:56,227 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:56,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:56,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:56,233 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:56,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:58,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:59,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:59,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:01,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:01,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:01,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:01,684 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:01,689 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:02,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:02,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:06,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:06,156 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:06,157 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:06,161 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:06,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:07,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:08,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:10,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:10,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:10,546 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:10,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:11,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:12,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:12,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:14,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:14,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:14,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:18,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:18,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:18,785 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:18,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:18,791 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:19,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:20,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:23,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:24,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:24,487 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:24,487 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:24,492 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:24,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:29:25,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:26,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:28,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:28,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:28,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:28,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:28,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:29,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:29,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:29,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:29,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:30,438 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:33,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:33,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:33,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:33,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:33,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:34,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:35,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:40,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:41,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:41,662 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:41,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:41,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:42,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:42,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:44,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:44,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:45,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:45,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:45,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:45,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:45,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:46,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:46,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:49,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:49,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:49,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:49,596 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:50,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:50,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:50,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:54,559 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:29:56,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:58,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:58,199 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:58,199 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:58,204 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:58,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:59,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:59,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:00,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:02,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:03,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:03,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:03,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:03,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:03,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:04,470 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:07,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:11,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:11,350 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:11,350 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:11,355 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:11,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:12,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:12,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:14,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:14,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:14,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:15,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:15,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:15,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:15,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:16,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:16,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:17,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:19,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:19,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:19,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:19,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:20,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:20,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:22,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:23,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:23,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:23,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:23,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:24,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:24,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:30:26,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:28,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:29,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:29,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:29,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:29,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:29,535 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:29,535 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:29,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:30,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:30,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:33,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:33,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:33,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:33,588 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:33,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:34,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:34,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:37,720 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:37,721 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:37,721 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:37,726 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:38,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:38,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:40,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:41,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:41,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:41,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:41,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:41,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:42,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:44,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:44,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:44,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:44,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:47,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:47,087 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:47,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:47,092 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:47,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:48,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:50,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:52,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:52,500 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:52,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:52,506 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:52,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:54,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:54,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:30:55,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:57,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:57,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:57,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:57,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:57,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:58,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:59,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:59,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:59,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:01,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:01,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:01,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:01,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:01,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:02,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:04,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:06,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:10,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:10,017 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:10,018 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:10,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:10,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:12,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:12,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:13,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:13,650 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:13,650 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:13,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:14,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:14,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:14,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:14,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:14,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:17,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:21,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:21,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:21,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:21,011 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:21,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:22,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:23,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:24,565 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:31:24,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:24,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:24,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:24,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:25,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:26,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:28,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:29,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:29,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:29,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:30,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:30,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:30,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:30,321 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:30,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:32,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:34,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:35,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:35,223 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:35,224 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:35,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:35,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:36,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:39,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:39,082 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:39,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:39,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:39,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:40,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:40,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:42,274 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:42,274 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:42,275 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:42,280 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:42,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:44,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:44,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:44,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:44,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:45,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:47,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:47,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:47,506 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:47,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:47,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:48,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:50,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:54,570 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:31:55,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:56,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:56,389 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:56,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:56,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:56,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:58,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:59,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:59,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:59,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:00,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:00,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:00,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:00,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:00,853 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:01,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:02,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:04,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:04,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:04,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:04,958 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:05,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:05,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:06,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:09,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:09,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:09,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:09,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:10,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:10,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:11,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:14,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:14,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:14,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:16,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:16,228 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:16,228 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:16,233 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:16,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:17,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:18,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:21,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:21,929 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:21,930 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:21,935 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:22,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:22,935 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:24,569 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:32:24,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:27,178 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:27,179 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:27,179 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:27,184 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:27,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:28,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:28,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:29,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:29,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:29,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:33,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:33,178 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:33,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:33,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:33,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:33,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:34,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:38,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:39,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:39,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:39,472 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:39,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:39,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:40,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:43,478 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:44,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:44,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:44,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:44,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:44,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:44,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:44,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:44,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:46,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:47,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:47,840 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:47,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:47,846 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:48,635 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:48,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:50,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:52,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:52,631 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:52,631 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:52,637 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:52,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:54,571 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:32:54,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:54,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:59,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:59,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:59,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:59,374 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:59,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:59,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:59,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:00,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:00,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:05,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:05,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:05,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:05,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:05,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:06,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:08,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:10,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:10,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:10,992 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:10,992 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:10,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:11,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:12,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:14,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:14,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:14,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:16,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:17,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:17,300 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:17,300 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:17,305 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:17,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:18,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:21,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:21,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:21,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:21,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:21,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:21,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:22,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:24,573 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:33:26,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:26,736 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:26,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:26,737 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:26,742 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:27,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:28,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:29,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:29,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:29,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:32,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:35,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:35,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:35,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:35,227 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:35,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:36,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:38,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:39,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:39,607 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:39,608 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:39,613 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:39,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:40,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:43,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:44,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:44,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:44,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:47,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:47,729 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:47,729 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:47,734 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:48,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:48,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:48,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:53,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:54,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:33:58,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:58,550 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:58,550 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:58,555 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:58,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:59,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:59,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:00,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:04,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:04,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:04,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:04,624 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:04,630 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:04,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:06,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:09,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:11,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:11,278 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:11,278 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:11,283 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:11,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:12,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:14,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:14,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:14,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:15,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:15,145 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:15,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:15,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:15,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:15,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:16,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:20,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:22,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:22,314 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:22,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:22,320 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:22,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:24,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:34:24,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:25,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:26,230 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:26,231 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:26,231 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:26,236 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:26,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:28,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:29,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:29,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:29,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:30,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:30,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:30,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:30,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:31,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:32,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:36,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:37,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:37,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:37,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:37,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:37,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:38,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:41,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:41,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:41,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:41,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:41,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:41,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:42,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:44,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:44,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:44,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:45,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:45,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:45,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:45,927 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:46,750 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:46,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:46,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:51,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:51,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:51,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:51,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:51,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:52,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:52,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:54,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:34:55,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:55,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:55,496 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:55,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:55,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:56,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:57,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:59,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:59,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:01,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:01,406 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:01,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:01,411 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:01,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:02,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:03,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:05,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:05,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:05,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:05,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:05,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:06,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:09,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:09,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:09,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:09,690 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:09,695 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:09,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:10,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:14,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:14,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:14,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:14,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:14,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:14,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:14,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:14,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:14,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:16,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:19,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:19,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:19,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:19,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:19,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:20,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:20,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:23,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:23,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:23,448 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:23,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:23,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:24,585 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:35:24,790 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:25,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:29,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:29,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:29,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:30,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:30,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:30,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:30,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:30,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:31,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:32,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:34,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:35,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:35,000 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:35,005 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:35,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:36,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:37,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:40,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:40,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:40,925 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:40,932 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:41,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:42,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:42,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:44,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:44,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:44,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:44,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:44,878 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:44,878 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:44,883 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:45,810 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:46,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:48,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:50,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:50,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:50,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:50,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:50,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:52,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:54,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:54,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:35:56,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:56,693 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:56,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:56,698 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:56,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:58,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:59,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:59,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:59,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:00,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:00,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:00,385 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:00,390 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:00,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:02,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:04,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:06,842 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:06,843 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:06,843 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:06,848 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:07,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:08,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:09,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:11,251 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:11,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:11,252 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:11,256 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:11,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:12,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:14,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:14,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:14,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:15,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:15,248 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:15,249 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:15,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:15,253 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:15,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:16,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:19,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:19,147 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:19,147 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:19,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:19,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:20,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:21,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:22,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:22,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:22,731 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:22,736 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:22,849 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:24,586 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:36:24,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:26,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:27,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:27,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:27,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:27,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:27,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:28,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:29,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:29,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:29,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:31,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:31,310 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:31,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:31,315 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:31,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:32,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:32,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:35,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:35,672 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:35,673 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:35,678 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:35,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:36,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:37,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:39,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:39,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:39,686 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:39,691 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:39,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:40,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:42,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:44,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:44,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:44,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:44,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:44,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:44,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:44,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:44,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:48,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:52,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:52,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:52,471 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:52,476 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:52,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:53,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:54,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:36:54,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:56,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:56,405 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:56,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:56,410 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:56,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:58,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:59,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:59,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:01,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:01,629 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:01,629 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:01,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:01,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:02,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:04,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:07,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:07,057 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:07,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:07,062 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:07,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:08,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:10,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:11,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:11,513 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:11,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:11,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:11,902 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:12,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:14,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:14,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:14,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:15,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:15,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:15,641 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:15,641 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:15,646 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:15,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:16,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:19,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:19,783 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:19,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:19,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:19,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:20,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:20,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:24,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:37:24,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:24,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:24,634 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:24,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:24,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:26,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:26,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:28,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:28,884 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:28,884 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:28,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:28,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:29,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:29,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:29,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:30,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:32,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:32,826 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:32,827 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:32,827 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:32,832 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:32,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:34,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:37,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:41,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:41,665 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:41,665 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:41,671 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:41,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:42,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:43,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:44,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:44,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:44,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:47,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:47,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:47,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:47,317 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:47,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:48,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:49,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:51,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:51,635 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:51,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:51,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:51,942 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:52,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:54,595 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:37:54,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:59,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:59,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:59,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:59,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:59,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:59,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:00,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:00,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:03,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:03,564 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:03,565 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:03,570 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:03,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:04,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:05,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:10,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:12,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:12,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:12,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:12,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:12,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:12,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:14,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:14,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:14,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:16,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:17,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:17,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:17,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:17,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:17,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:18,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:21,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:21,934 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:21,934 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:21,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:21,939 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:21,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:22,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:24,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:38:25,751 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:25,752 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:25,752 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:25,758 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:25,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:26,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:27,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:29,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:29,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:29,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:31,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:31,821 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:31,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:31,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:31,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:32,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:32,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:36,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:36,156 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:36,156 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:36,161 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:36,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:36,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:38,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:41,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:41,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:41,239 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:41,244 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:41,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:42,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:43,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:44,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:44,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:44,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:46,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:46,547 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:46,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:46,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:46,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:48,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:48,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:51,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:51,408 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:51,408 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:51,414 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:52,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:53,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:54,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:54,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:38:56,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:56,014 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:56,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:56,020 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:57,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:57,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:59,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:00,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:02,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:02,158 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:02,158 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:02,163 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:03,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:03,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:06,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:06,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:06,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:06,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:06,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:07,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:07,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:09,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:09,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:09,682 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:09,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:10,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:11,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:11,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:13,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:13,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:13,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:13,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:14,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:14,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:14,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:14,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:15,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:17,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:22,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:22,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:22,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:22,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:23,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:23,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:24,598 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:39:25,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:27,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:27,507 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:27,507 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:27,512 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:28,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:28,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:29,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:29,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:29,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:29,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:31,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:31,478 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:31,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:31,483 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:32,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:33,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:33,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:36,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:36,793 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:36,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:36,799 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:37,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:38,799 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:39,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:41,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:41,301 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:41,301 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:41,306 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:42,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:43,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:44,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:44,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:44,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:44,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:45,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:45,261 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:45,262 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:45,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:46,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:47,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:48,958 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:48,959 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:48,959 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:48,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:49,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:49,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:51,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:54,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:54,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:54,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:54,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:54,600 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:39:55,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:55,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:57,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:59,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:59,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:59,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:01,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:03,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:03,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:03,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:03,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:04,075 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:05,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:07,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:07,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:07,322 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:07,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:07,327 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:08,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:09,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:11,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:11,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:11,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:11,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:12,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:12,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:13,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:14,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:14,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:14,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:15,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:15,529 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:15,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:15,534 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:16,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:17,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:17,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:21,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:21,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:21,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:21,788 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:22,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:22,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:23,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:24,602 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:40:26,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:26,051 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:26,051 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:26,056 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:26,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:27,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:28,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:29,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:29,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:29,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:30,416 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:30,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:30,417 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:30,422 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:31,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:33,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:33,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:34,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:34,688 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:34,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:34,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:35,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:37,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:38,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:40,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:40,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:40,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:40,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:41,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:41,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:44,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:44,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:44,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:44,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:44,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:44,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:44,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:44,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:45,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:47,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:49,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:51,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:51,161 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:51,161 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:51,166 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:52,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:53,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:54,604 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:40:54,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:55,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:55,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:55,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:55,374 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:56,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:57,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:59,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:59,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:59,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:59,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:59,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:59,348 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:59,353 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:00,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:00,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:01,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:03,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:03,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:03,536 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:03,541 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:04,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:05,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:05,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:10,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:10,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:10,413 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:10,418 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:11,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:11,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:13,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:14,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:14,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:14,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:15,034 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:15,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:15,035 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:15,039 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:15,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:17,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:17,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:19,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:19,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:19,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:19,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:20,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:21,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:22,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:23,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:23,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:23,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:23,802 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:24,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:24,609 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:41:25,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:27,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:27,334 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:27,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:27,339 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:28,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:28,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:29,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:29,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:29,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:29,168 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:32,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:32,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:32,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:32,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:32,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:33,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:34,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:37,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:37,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:37,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:37,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:38,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:39,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:39,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:41,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:41,557 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:41,558 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:41,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:42,180 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:43,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:44,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:44,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:44,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:45,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:46,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:46,961 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:46,962 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:46,966 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:47,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:49,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:51,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:51,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:51,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:51,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:51,505 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:52,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:53,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:54,608 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:41:56,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:59,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:59,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:59,009 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:59,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:59,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:59,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:01,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:02,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:03,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:03,828 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:03,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:03,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:04,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:05,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:07,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:09,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:09,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:09,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:09,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:10,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:11,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:13,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:13,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:13,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:13,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:13,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:13,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:14,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:14,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:15,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:16,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:16,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:16,795 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:16,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:17,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:18,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:19,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:20,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:20,603 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:20,603 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:20,608 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:21,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:23,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:24,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:24,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:42:29,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:29,024 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:29,025 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:29,029 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:29,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:29,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:29,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:29,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:30,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:31,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:34,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:34,321 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:34,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:34,325 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:35,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:35,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:35,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:40,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:43,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:43,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:43,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:43,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:43,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:44,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:44,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:44,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:45,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:46,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:47,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:47,015 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:47,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:47,020 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:47,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:49,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:52,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:52,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:52,126 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:52,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:52,132 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:52,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:53,253 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:54,612 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:42:56,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:56,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:56,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:56,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:57,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:57,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:57,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:59,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:01,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:01,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:01,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:01,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:02,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:02,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:03,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:05,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:05,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:05,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:05,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:06,263 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:07,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:07,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:09,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:09,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:09,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:09,535 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:10,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:11,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:13,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:14,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:14,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:14,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:14,738 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:14,738 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:14,743 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:15,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:17,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:18,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:21,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:21,227 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:21,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:21,232 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:21,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:23,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:24,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:24,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:43:26,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:26,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:26,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:26,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:26,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:27,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:29,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:29,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:29,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:29,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:32,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:32,600 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:32,600 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:32,605 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:33,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:34,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:35,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:39,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:41,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:41,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:41,627 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:41,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:42,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:43,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:44,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:44,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:44,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:45,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:50,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:54,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:43:55,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:56,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:56,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:56,190 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:56,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:56,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:57,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:59,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:59,935 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:59,935 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:59,936 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:59,941 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:00,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:00,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:01,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:03,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:03,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:03,989 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:03,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:04,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:05,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:05,995 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:09,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:09,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:09,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:09,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:09,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:11,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:11,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:14,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:14,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:16,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:16,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:16,302 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:16,303 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:16,307 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:16,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:17,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:21,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:22,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:22,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:22,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:22,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:22,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:23,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:24,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:44:25,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:25,990 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:25,990 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:25,995 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:26,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:26,995 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:27,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:29,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:29,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:29,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:29,736 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:29,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:29,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:29,741 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:30,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:31,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:32,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:33,481 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:33,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:33,482 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:33,487 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:34,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:35,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:37,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:37,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:37,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:37,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:38,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:38,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:39,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:42,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:42,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:42,432 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:42,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:43,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:43,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:44,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:44,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:44,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:44,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:46,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:46,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:46,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:46,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:47,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:49,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:49,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:51,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:51,061 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:51,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:51,067 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:51,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:53,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:54,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:44:55,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:56,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:56,062 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:56,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:56,067 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:56,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:57,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:59,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:01,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:02,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:02,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:02,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:02,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:03,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:03,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:06,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:07,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:07,008 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:07,008 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:07,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:07,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:09,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:10,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:10,601 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:10,601 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:10,606 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:11,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:11,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:12,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:14,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:14,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:15,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:15,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:15,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:15,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:15,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:17,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:18,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:23,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:24,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:45:24,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:24,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:24,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:24,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:25,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:25,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:28,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:29,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:29,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:29,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:29,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:29,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:29,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:29,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:29,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:31,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:34,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:34,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:34,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:34,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:34,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:35,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:37,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:39,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:40,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:40,175 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:40,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:40,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:40,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:41,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:44,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:44,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:44,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:44,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:44,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:44,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:44,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:44,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:45,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:45,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:48,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:48,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:48,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:48,252 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:48,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:49,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:51,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:53,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:53,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:53,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:53,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:53,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:54,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:45:55,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:56,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:58,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:58,073 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:58,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:58,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:58,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:59,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:59,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:59,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:59,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:02,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:04,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:04,945 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:04,945 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:04,950 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:05,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:07,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:07,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:10,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:10,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:10,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:10,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:10,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:11,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:13,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:13,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:13,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:13,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:13,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:14,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:14,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:14,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:14,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:15,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:17,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:17,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:17,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:17,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:17,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:18,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:19,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:21,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:21,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:21,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:21,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:21,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:23,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:23,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:24,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:46:26,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:26,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:26,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:26,439 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:26,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:27,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:28,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:29,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:29,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:29,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:29,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:29,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:29,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:29,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:30,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:31,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:34,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:34,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:34,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:34,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:34,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:35,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:35,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:39,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:41,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:41,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:41,142 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:41,147 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:41,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:43,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:44,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:44,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:44,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:45,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:46,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:46,125 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:46,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:46,130 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:46,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:47,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:49,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:49,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:49,813 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:49,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:50,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:50,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:51,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:54,629 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:46:55,254 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:55,255 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:55,255 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:55,260 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:55,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:56,260 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:57,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:59,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:59,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:59,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:59,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:59,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:59,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:59,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:59,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:01,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:01,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:03,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:03,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:03,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:03,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:04,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:05,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:06,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:07,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:07,534 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:07,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:07,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:08,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:09,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:11,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:11,437 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:11,437 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:11,443 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:11,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:12,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:13,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:14,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:14,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:14,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:15,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:15,581 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:15,581 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:15,586 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:16,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:17,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:17,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:20,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:20,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:20,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:20,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:21,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:21,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:22,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:24,631 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:47:24,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:24,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:24,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:24,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:25,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:27,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:27,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:28,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:28,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:28,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:28,321 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:28,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:29,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:29,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:29,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:29,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:33,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:33,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:33,973 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:33,973 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:33,978 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:34,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:35,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:39,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:40,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:40,082 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:40,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:40,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:40,546 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:41,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:44,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:44,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:44,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:46,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:46,507 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:46,508 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:46,512 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:46,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:47,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:49,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:54,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:54,633 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:47:54,904 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:54,905 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:54,905 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:54,910 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:55,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:57,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:58,436 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:58,437 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:58,437 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:58,442 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:58,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:59,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:59,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:59,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:00,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:02,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:02,612 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:02,612 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:02,617 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:03,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:03,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:05,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:05,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:05,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:05,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:05,884 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:06,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:07,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:10,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:10,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:10,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:10,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:10,692 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:11,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:11,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:14,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:14,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:14,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:14,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:14,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:14,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:14,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:15,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:16,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:18,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:18,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:18,471 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:18,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:18,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:19,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:22,478 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:23,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:23,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:23,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:23,006 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:23,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:24,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:48:25,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:26,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:26,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:26,974 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:26,979 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:27,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:27,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:29,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:29,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:29,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:29,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:32,251 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:32,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:32,252 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:32,257 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:32,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:33,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:33,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:37,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:37,522 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:37,522 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:37,527 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:37,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:38,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:39,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:41,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:41,302 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:41,302 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:41,307 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:41,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:43,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:44,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:44,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:44,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:45,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:45,342 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:45,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:45,347 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:45,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:47,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:49,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:50,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:50,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:50,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:50,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:50,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:51,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:54,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:48:54,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:54,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:54,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:54,778 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:54,783 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:55,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:55,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:58,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:58,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:58,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:58,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:59,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:59,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:59,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:59,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:00,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:02,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:02,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:02,432 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:02,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:02,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:03,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:05,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:07,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:07,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:07,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:07,799 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:08,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:09,642 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:10,800 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:12,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:12,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:12,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:12,330 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:12,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:13,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:14,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:14,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:16,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:18,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:18,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:18,664 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:18,669 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:19,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:19,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:21,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:23,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:23,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:23,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:23,092 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:23,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:24,643 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:49:25,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:27,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:28,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:28,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:28,962 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:28,967 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:29,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:29,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:29,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:29,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:31,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:33,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:35,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:35,718 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:35,718 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:35,723 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:36,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:37,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:38,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:42,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:42,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:42,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:42,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:43,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:43,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:43,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:44,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:44,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:44,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:49,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:51,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:51,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:51,207 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:51,212 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:51,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:53,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:54,642 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:49:54,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:55,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:55,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:55,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:55,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:55,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:57,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:59,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:59,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:00,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:01,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:01,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:01,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:01,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:02,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:03,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:05,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:08,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:08,189 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:08,190 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:08,194 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:08,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:09,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:11,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:12,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:12,914 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:12,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:12,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:13,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:13,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:14,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:14,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:16,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:17,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:17,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:17,805 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:17,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:18,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:19,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:21,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:21,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:21,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:21,105 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:21,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:22,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:23,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:24,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:50:25,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:25,664 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:25,664 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:25,669 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:25,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:27,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:27,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:29,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:29,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:29,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:29,452 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:29,452 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:29,452 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:29,457 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:29,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:31,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:33,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:33,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:33,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:33,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:33,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:33,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:35,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:37,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:37,109 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:37,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:37,114 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:37,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:39,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:39,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:41,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:41,226 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:41,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:41,232 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:41,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:43,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:44,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:44,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:44,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:44,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:46,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:46,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:46,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:46,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:46,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:47,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:49,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:50,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:50,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:50,692 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:50,697 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:50,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:51,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:54,646 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:50:55,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:55,248 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:55,249 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:55,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:55,253 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:55,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:57,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:59,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:59,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:59,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:00,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:01,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:01,133 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:01,133 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:01,138 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:01,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:03,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:05,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:05,705 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:05,705 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:05,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:05,710 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:05,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:07,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:10,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:10,630 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:10,630 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:10,635 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:10,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:11,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:11,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:14,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:14,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:14,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:15,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:15,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:15,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:15,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:15,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:17,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:17,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:19,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:19,845 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:19,845 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:19,850 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:20,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:21,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:22,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:23,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:23,785 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:23,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:23,790 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:24,648 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:51:24,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:25,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:28,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:29,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:29,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:29,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:31,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:31,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:31,919 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:31,924 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:32,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:33,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:33,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:37,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:37,175 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:37,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:37,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:37,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:39,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:39,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:40,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:40,972 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:40,973 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:40,977 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:41,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:41,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:44,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:44,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:44,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:44,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:45,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:45,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:45,726 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:45,731 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:45,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:47,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:49,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:49,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:49,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:49,974 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:49,979 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:50,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:51,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:54,653 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:51:55,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:56,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:56,272 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:56,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:56,278 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:56,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:57,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:59,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:59,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:59,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:59,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:59,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:59,962 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:59,967 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:00,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:01,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:01,824 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:03,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:03,321 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:03,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:03,327 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:03,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:05,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:06,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:11,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:11,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:11,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:11,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:11,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:12,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:13,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:14,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:14,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:14,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:16,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:16,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:16,646 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:16,647 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:16,652 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:16,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:17,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:21,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:22,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:22,632 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:22,632 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:22,638 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:22,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:23,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:24,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:52:26,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:26,494 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:26,494 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:26,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:26,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:27,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:27,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:29,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:29,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:29,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:31,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:31,004 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:31,004 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:31,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:31,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:31,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:33,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:37,460 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:37,461 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:37,461 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:37,466 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:37,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:38,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:39,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:42,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:42,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:42,554 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:42,559 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:42,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:43,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:43,867 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:44,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:44,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:44,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:47,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:47,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:47,291 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:47,296 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:47,870 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:49,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:49,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:54,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:54,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:52:55,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:55,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:55,726 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:55,731 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:55,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:57,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:59,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:59,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:59,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:59,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:59,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:59,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:59,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:59,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:59,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:01,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:03,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:03,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:03,138 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:03,143 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:03,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:03,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:05,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:07,724 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:07,725 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:07,725 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:07,730 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:07,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:09,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:10,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:11,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:11,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:11,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:11,504 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:11,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:13,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:14,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:14,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:14,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:16,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:16,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:16,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:16,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:16,239 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:16,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:17,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:21,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:24,656 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:53:24,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:24,867 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:24,867 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:24,872 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:24,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:25,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:26,872 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:28,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:28,654 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:28,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:28,661 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:28,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:29,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:29,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:29,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:29,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:32,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:32,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:32,690 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:32,691 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:32,696 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:32,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:33,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:37,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:42,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:42,519 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:42,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:42,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:42,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:43,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:43,927 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:44,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:44,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:44,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:47,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:47,147 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:47,147 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:47,152 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:47,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:47,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:49,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:51,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:51,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:51,148 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:51,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:51,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:51,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:54,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:53:54,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:56,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:56,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:56,378 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:56,383 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:56,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:57,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:59,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:59,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:59,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:00,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:01,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:01,611 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:01,611 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:01,616 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:01,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:03,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:05,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:07,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:07,934 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:07,935 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:07,939 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:07,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:09,954 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:10,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:13,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:13,102 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:13,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:13,107 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:13,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:13,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:14,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:14,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:14,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:16,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:17,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:17,633 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:17,634 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:17,639 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:17,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:19,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:21,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:22,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:22,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:22,556 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:22,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:22,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:23,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:24,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:54:27,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:29,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:29,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:29,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:31,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:31,098 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:31,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:31,103 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:31,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:31,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:33,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:35,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:35,295 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:35,295 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:35,300 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:35,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:35,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:38,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:40,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:40,357 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:40,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:40,362 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:40,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:41,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:43,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:44,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:44,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:44,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:45,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:45,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:45,348 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:45,353 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:45,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:47,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:48,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:50,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:50,061 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:50,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:50,066 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:50,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:51,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:54,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:54,662 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:54:59,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:59,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:59,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:59,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:59,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:59,825 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:59,826 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:59,831 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:00,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:02,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:04,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:07,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:07,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:07,554 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:07,559 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:08,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:10,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:10,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:12,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:12,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:12,989 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:12,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:13,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:14,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:14,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:14,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:14,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:16,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:18,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:18,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:18,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:18,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:19,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:20,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:22,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:23,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:23,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:23,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:23,584 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:24,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:24,664 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:55:26,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:27,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:29,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:29,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:29,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:29,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:29,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:29,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:29,550 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:30,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:32,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:33,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:33,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:33,148 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:33,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:33,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:34,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:34,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:37,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:37,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:37,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:37,202 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:38,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:38,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:38,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:41,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:41,602 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:41,603 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:41,608 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:42,043 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:43,608 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:44,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:44,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:44,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:44,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:45,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:45,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:45,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:45,230 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:46,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:46,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:49,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:49,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:49,713 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:49,713 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:49,721 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:50,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:52,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:54,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:55:54,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:58,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:58,458 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:58,458 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:58,463 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:59,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:59,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:59,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:59,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:00,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:00,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:03,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:03,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:03,204 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:03,209 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:04,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:04,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:06,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:09,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:09,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:09,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:09,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:10,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:11,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:12,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:14,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:14,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:14,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:14,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:14,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:14,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:14,317 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:15,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:16,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:17,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:19,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:19,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:19,145 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:19,149 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:20,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:20,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:23,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:23,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:23,169 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:23,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:23,174 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:24,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:24,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:24,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:56:26,856 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:26,857 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:26,857 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:26,862 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:27,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:28,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:28,863 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:29,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:29,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:29,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:32,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:32,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:32,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:32,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:33,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:34,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:34,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:37,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:37,855 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:37,855 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:37,860 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:38,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:39,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:40,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:42,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:42,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:42,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:42,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:43,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:44,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:44,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:44,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:44,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:45,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:47,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:47,882 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:47,882 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:47,887 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:48,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:50,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:50,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:52,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:52,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:52,468 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:52,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:53,113 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:54,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:54,670 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:56:56,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:56,340 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:56,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:56,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:56,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:57,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:58,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:59,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:59,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:59,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:59,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:59,935 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:59,935 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:59,940 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:00,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:01,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:02,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:03,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:03,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:03,257 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:03,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:04,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:04,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:07,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:07,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:07,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:07,394 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:07,400 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:08,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:08,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:12,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:13,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:13,270 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:13,270 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:13,276 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:14,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:14,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:14,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:14,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:14,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:17,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:17,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:17,740 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:17,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:17,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:18,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:20,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:21,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:21,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:21,731 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:21,736 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:22,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:23,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:24,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:24,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:57:26,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:26,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:26,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:26,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:26,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:28,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:29,027 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:29,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:29,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:29,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:30,792 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:30,793 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:30,793 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:30,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:31,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:32,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:34,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:34,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:34,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:34,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:34,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:35,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:36,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:38,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:38,468 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:38,469 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:38,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:39,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:40,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:40,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:42,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:42,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:42,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:42,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:43,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:44,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:44,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:44,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:46,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:47,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:47,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:47,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:47,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:48,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:48,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:51,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:51,611 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:51,611 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:51,611 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:51,616 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:52,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:54,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:54,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:57:56,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:59,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:59,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:59,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:59,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:59,322 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:59,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:59,326 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:00,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:00,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:02,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:02,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:02,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:02,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:02,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:03,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:04,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:06,792 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:06,793 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:06,793 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:06,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:07,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:07,798 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:08,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:11,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:11,470 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:11,471 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:11,476 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:12,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:12,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:13,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:14,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:14,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:14,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:14,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:14,918 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:14,918 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:14,923 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:15,199 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:16,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:18,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:19,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:19,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:19,135 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:19,140 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:19,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:20,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:24,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:24,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:58:24,855 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:24,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:24,856 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:24,861 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:25,209 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:26,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:28,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:28,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:28,849 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:28,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:29,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:29,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:29,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:29,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:29,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:30,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:32,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:32,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:32,923 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:32,928 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:33,225 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:34,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:34,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:37,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:37,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:37,453 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:37,459 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:38,232 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:38,232 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:40,460 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:41,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:41,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:41,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:41,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:41,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:42,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:44,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:44,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:45,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:45,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:45,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:45,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:45,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:46,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:48,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:50,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:50,329 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:50,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:50,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:51,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:51,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:52,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:54,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:58:56,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:56,824 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:56,825 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:56,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:56,830 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:57,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:58,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:59,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:59,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:59,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:00,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:00,490 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:00,490 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:00,495 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:01,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:02,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:02,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:06,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:06,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:06,363 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:06,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:07,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:08,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:08,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:10,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:10,336 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:10,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:10,341 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:11,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:12,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:13,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:13,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:13,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:13,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:13,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:14,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:14,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:14,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:14,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:16,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:18,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:18,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:18,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:18,105 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:18,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:19,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:20,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:23,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:23,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:23,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:23,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:23,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:24,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:24,680 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:59:24,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:26,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:26,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:26,682 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:26,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:27,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:28,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:29,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:29,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:29,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:30,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:32,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:32,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:32,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:32,596 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:33,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:34,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:35,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:37,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:37,040 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:37,040 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:37,045 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:37,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:38,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:40,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:40,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:40,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:40,652 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:40,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:41,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:42,299 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:44,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:44,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:45,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:45,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:45,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:45,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:45,746 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:46,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:48,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:50,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:50,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:50,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:50,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:50,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:51,152 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:52,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:53,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:53,680 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:53,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:53,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:54,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:54,683 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:59:56,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:56,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:57,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:57,551 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:57,551 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:57,556 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:58,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:58,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:59,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:59,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:59,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:02,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:03,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:03,457 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:03,458 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:03,463 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:04,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:04,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:07,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:10,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:10,568 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:10,569 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:10,574 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:11,328 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:12,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:12,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:14,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:14,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:14,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:15,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:15,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:15,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:15,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:15,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:16,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:18,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:19,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:19,441 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:19,441 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:19,446 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:20,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:20,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:23,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:23,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:23,707 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:23,707 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:23,712 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:24,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:24,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:00:26,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:27,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:27,616 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:27,616 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:27,621 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:28,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:28,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:28,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:29,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:29,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:29,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:33,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:33,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:33,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:33,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:33,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:34,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:34,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:38,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:38,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:38,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:38,269 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:38,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:39,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:40,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:43,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:43,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:43,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:43,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:44,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:44,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:44,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:44,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:49,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:49,206 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:49,206 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:49,211 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:49,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:50,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:50,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:53,042 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:53,043 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:53,043 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:53,048 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:53,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:54,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:54,690 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:00:55,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:56,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:56,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:56,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:56,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:57,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:58,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:59,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:59,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:59,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:01,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:02,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:02,884 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:02,884 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:02,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:03,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:04,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:06,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:06,604 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:06,604 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:06,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:06,609 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:07,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:08,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:10,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:10,162 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:10,162 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:10,167 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:10,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:12,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:12,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:14,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:14,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:14,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:14,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:14,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:14,304 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:14,309 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:14,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:16,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:17,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:17,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:17,664 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:17,664 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:17,669 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:18,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:18,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:22,670 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:24,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:01:25,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:25,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:25,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:25,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:25,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:26,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:28,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:29,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:29,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:29,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:30,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:30,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:30,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:30,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:31,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:32,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:33,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:34,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:34,608 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:34,608 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:34,613 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:35,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:36,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:39,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:39,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:39,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:39,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:39,292 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:39,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:40,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:42,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:42,543 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:42,543 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:42,548 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:43,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:44,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:44,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:44,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:44,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:48,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:48,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:48,192 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:48,197 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:48,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:50,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:50,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:52,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:52,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:52,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:52,504 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:53,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:54,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:54,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:01:55,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:56,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:56,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:56,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:56,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:57,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:58,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:59,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:59,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:59,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:01,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:02,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:02,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:02,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:02,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:03,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:04,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:06,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:08,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:08,789 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:08,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:08,794 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:09,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:10,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:11,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:13,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:13,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:13,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:13,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:13,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:14,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:14,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:14,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:14,489 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:16,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:16,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:16,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:16,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:16,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:17,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:18,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:20,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:20,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:20,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:20,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:21,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:22,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:22,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:24,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:24,415 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:24,415 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:24,420 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:24,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:24,693 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:02:26,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:28,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:29,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:29,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:29,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:29,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:29,392 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:29,392 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:29,397 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:29,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:30,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:32,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:32,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:32,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:32,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:33,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:34,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:34,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:38,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:38,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:38,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:38,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:39,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:40,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:40,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:42,281 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:42,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:42,282 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:42,287 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:42,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:44,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:44,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:44,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:44,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:46,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:46,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:46,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:46,370 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:46,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:46,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:48,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:50,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:50,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:50,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:50,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:50,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:52,180 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:52,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:54,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:54,321 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:54,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:54,327 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:54,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:54,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:02:56,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:57,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:58,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:58,791 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:58,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:58,796 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:59,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:59,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:59,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:59,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:00,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:03,180 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:03,181 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:03,181 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:03,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:03,186 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:03,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:04,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:07,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:07,164 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:07,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:07,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:07,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:08,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:09,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:12,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:12,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:12,265 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:12,270 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:12,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:14,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:14,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:14,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:14,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:14,578 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:17,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:17,581 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:17,581 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:17,586 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:18,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:18,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:19,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:21,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:21,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:21,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:21,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:21,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:22,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:24,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:03:24,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:25,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:25,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:25,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:25,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:26,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:26,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:29,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:29,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:29,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:30,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:34,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:34,125 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:34,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:34,131 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:34,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:36,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:36,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:39,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:39,757 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:39,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:39,762 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:40,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:40,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:41,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:44,058 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:44,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:44,063 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:44,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:44,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:44,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:46,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:47,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:48,180 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:48,180 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:48,181 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:48,185 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:48,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:50,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:52,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:52,852 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:52,852 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:52,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:52,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:53,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:54,637 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:54,699 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:03:57,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:57,838 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:57,838 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:57,843 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:58,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:58,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:58,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:59,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:59,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:59,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:04,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:06,998 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:06,999 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:06,999 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:07,004 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:07,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:08,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:10,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:12,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:12,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:12,214 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:12,220 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:12,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:14,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:14,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:14,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:14,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:15,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:16,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:16,219 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:16,219 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:16,224 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:16,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:18,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:20,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:20,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:20,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:20,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:20,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:20,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:22,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:24,705 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:04:25,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:27,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:27,257 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:27,258 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:27,263 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:27,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:28,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:29,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:29,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:29,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:31,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:33,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:33,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:33,546 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:33,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:33,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:34,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:36,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:41,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:42,025 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:42,026 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:42,026 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:42,032 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:42,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:44,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:44,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:44,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:47,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:50,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:50,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:50,363 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:50,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:50,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:52,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:52,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:54,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:04:54,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:54,819 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:54,819 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:54,825 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:55,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:56,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:57,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:58,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:58,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:58,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:58,813 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:59,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:59,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:59,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:59,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:00,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:03,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:04,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:04,022 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:04,022 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:04,027 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:04,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:06,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:08,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:08,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:08,008 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:08,013 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:08,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:08,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:09,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:14,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:14,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:14,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:14,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:14,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:14,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:14,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:14,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:15,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:16,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:19,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:21,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:21,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:21,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:21,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:21,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:22,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:24,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:05:25,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:25,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:25,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:25,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:25,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:25,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:26,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:29,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:29,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:29,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:31,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:32,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:32,422 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:32,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:32,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:32,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:34,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:36,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:36,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:36,834 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:36,834 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:36,839 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:37,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:38,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:41,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:44,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:44,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:45,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:45,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:45,173 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:45,178 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:45,789 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:46,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:47,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:50,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:50,654 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:50,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:50,660 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:50,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:52,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:52,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:54,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:05:56,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:56,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:56,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:56,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:56,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:57,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:58,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:59,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:59,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:59,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:00,911 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:00,911 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:00,912 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:00,916 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:01,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:02,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:02,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:05,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:05,491 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:05,492 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:05,496 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:05,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:06,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:08,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:13,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:13,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:13,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:13,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:13,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:14,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:14,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:14,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:14,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:14,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:18,194 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:18,195 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:18,195 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:18,200 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:18,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:19,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:20,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:22,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:22,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:22,250 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:22,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:22,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:24,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:24,710 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:06:24,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:26,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:26,430 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:26,431 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:26,435 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:26,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:28,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:29,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:29,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:29,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:29,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:34,408 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:34,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:34,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:34,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:34,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:34,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:36,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:39,700 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:40,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:40,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:40,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:40,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:40,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:42,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:44,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:44,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:44,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:44,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:44,670 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:44,670 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:44,675 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:44,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:44,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:46,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:49,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:53,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:53,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:53,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:53,210 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:53,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:54,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:06:54,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:55,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:59,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:59,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:59,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:01,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:01,496 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:01,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:01,497 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:01,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:01,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:02,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:05,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:05,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:05,928 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:05,933 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:06,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:06,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:06,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:11,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:12,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:12,705 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:12,705 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:12,710 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:12,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:14,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:14,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:14,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:14,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:16,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:16,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:16,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:16,344 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:16,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:17,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:18,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:21,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:21,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:21,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:21,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:21,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:22,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:23,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:24,714 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:07:25,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:25,255 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:25,256 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:25,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:25,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:26,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:28,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:29,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:29,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:29,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:32,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:32,641 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:32,642 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:32,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:32,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:33,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:34,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:36,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:36,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:36,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:36,383 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:36,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:38,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:39,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:40,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:40,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:40,555 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:40,560 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:40,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:42,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:44,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:44,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:44,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:45,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:49,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:49,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:49,176 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:49,181 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:49,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:50,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:51,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:53,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:53,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:53,562 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:53,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:53,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:54,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:07:54,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:56,720 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:58,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:58,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:58,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:58,210 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:58,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:58,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:59,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:59,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:59,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:02,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:02,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:02,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:02,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:02,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:02,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:02,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:06,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:06,121 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:06,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:06,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:06,927 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:06,927 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:08,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:10,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:10,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:10,962 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:10,967 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:11,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:12,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:13,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:14,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:14,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:14,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:14,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:14,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:14,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:14,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:14,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:16,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:19,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:20,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:20,147 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:20,147 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:20,152 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:20,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:20,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:23,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:23,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:23,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:23,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:23,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:24,718 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:08:24,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:25,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:27,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:27,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:27,833 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:27,838 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:27,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:28,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:29,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:29,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:29,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:31,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:31,132 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:31,132 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:31,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:31,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:31,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:32,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:35,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:35,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:35,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:35,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:35,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:36,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:36,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:38,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:38,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:38,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:38,958 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:38,959 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:40,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:41,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:43,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:43,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:43,190 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:43,195 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:43,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:44,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:44,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:44,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:44,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:47,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:48,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:48,359 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:48,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:48,365 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:48,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:50,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:52,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:53,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:53,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:53,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:53,150 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:53,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:54,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:08:54,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:57,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:57,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:57,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:57,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:57,578 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:57,979 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:58,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:59,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:59,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:59,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:02,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:02,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:02,588 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:02,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:02,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:02,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:04,987 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:07,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:08,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:08,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:08,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:08,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:08,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:10,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:12,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:12,395 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:12,395 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:12,400 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:12,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:13,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:14,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:14,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:14,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:14,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:17,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:17,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:17,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:17,109 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:17,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:19,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:19,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:21,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:21,955 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:21,955 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:21,960 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:22,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:23,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:24,722 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:09:24,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:26,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:26,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:26,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:26,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:27,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:27,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:29,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:29,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:29,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:29,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:29,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:29,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:29,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:29,773 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:30,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:31,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:33,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:33,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:33,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:33,813 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:34,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:34,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:35,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:38,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:38,109 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:38,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:38,114 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:39,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:39,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:40,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:41,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:41,980 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:41,980 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:41,986 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:42,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:43,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:44,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:44,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:44,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:45,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:47,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:47,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:47,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:47,420 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:48,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:49,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:50,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:51,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:51,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:51,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:51,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:52,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:53,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:54,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:09:55,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:59,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:59,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:59,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:59,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:59,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:59,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:59,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:00,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:00,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:01,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:03,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:03,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:03,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:03,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:04,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:05,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:06,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:07,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:07,914 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:07,914 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:07,919 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:08,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:09,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:12,920 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:14,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:14,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:14,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:16,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:16,380 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:16,380 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:16,385 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:17,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:18,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:19,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:19,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:19,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:19,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:19,883 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:20,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:21,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:23,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:24,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:10:25,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:25,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:25,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:25,557 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:26,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:27,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:29,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:29,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:29,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:29,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:30,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:30,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:30,616 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:30,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:31,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:33,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:34,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:34,313 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:34,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:34,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:35,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:35,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:35,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:39,010 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:39,011 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:39,011 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:39,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:39,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:41,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:41,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:43,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:43,694 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:43,694 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:43,699 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:44,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:44,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:44,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:44,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:45,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:46,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:48,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:48,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:48,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:48,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:49,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:51,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:51,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:54,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:54,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:54,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:54,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:54,728 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:10:55,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:57,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:57,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:59,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:59,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:59,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:59,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:59,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:59,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:59,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:00,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:01,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:03,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:03,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:03,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:03,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:03,440 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:04,113 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:05,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:08,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:12,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:12,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:12,224 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:12,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:13,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:13,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:14,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:14,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:14,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:14,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:16,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:16,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:16,768 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:16,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:17,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:19,128 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:19,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:21,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:21,208 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:21,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:21,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:22,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:23,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:24,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:11:24,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:24,740 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:24,740 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:24,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:25,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:25,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:27,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:29,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:29,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:29,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:30,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:30,838 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:30,838 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:30,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:30,843 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:31,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:33,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:34,487 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:34,488 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:34,488 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:34,493 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:35,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:36,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:37,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:37,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:37,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:37,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:37,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:38,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:39,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:41,910 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:43,442 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:43,443 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:43,443 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:43,448 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:44,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:44,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:44,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:44,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:45,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:47,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:47,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:47,569 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:47,570 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:47,574 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:48,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:49,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:52,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:52,831 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:52,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:52,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:52,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:53,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:54,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:11:55,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:56,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:56,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:56,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:56,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:57,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:58,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:59,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:59,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:59,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:59,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:01,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:01,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:01,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:01,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:02,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:03,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:04,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:05,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:05,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:05,508 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:05,513 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:06,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:07,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:09,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:10,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:10,229 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:10,229 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:10,234 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:11,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:11,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:14,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:14,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:14,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:15,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:15,162 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:15,162 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:15,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:15,167 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:15,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:17,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:18,628 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:18,629 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:18,629 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:18,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:19,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:20,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:21,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:24,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:24,730 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:24,730 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:24,736 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:24,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:12:25,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:25,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:27,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:28,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:28,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:28,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:28,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:29,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:29,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:29,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:29,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:29,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:31,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:33,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:33,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:33,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:33,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:34,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:35,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:36,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:37,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:37,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:37,656 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:37,661 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:38,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:39,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:41,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:41,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:41,167 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:41,172 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:41,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:42,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:43,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:44,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:44,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:44,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:45,539 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:45,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:45,540 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:45,545 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:46,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:47,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:47,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:49,254 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:49,255 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:49,255 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:49,260 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:50,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:51,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:53,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:54,738 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:12:55,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:55,405 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:55,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:55,413 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:56,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:57,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:58,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:59,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:59,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:59,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:59,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:59,838 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:59,838 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:59,843 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:00,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:01,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:03,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:03,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:03,660 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:03,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:03,665 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:04,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:05,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:08,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:08,084 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:08,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:08,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:08,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:09,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:09,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:14,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:14,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:14,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:14,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:14,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:14,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:14,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:14,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:15,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:17,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:19,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:19,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:19,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:19,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:19,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:19,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:21,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:24,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:24,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:24,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:24,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:24,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:24,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:24,743 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:13:25,263 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:28,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:28,616 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:28,616 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:28,621 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:29,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:29,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:29,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:29,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:29,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:31,269 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:32,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:32,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:32,364 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:32,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:33,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:33,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:34,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:35,968 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:35,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:35,969 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:35,974 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:36,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:37,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:39,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:40,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:40,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:40,018 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:40,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:40,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:41,280 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:44,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:44,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:44,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:44,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:44,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:44,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:44,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:45,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:45,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:45,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:50,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:52,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:52,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:52,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:52,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:53,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:53,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:54,743 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:13:55,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:58,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:58,424 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:58,425 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:58,429 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:59,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:59,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:59,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:59,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:59,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:01,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:01,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:01,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:01,824 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:01,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:02,299 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:03,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:05,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:05,884 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:05,884 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:05,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:06,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:06,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:07,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:09,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:09,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:09,434 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:09,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:10,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:11,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:12,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:14,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:14,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:14,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:14,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:14,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:14,517 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:14,522 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:15,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:15,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:17,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:18,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:18,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:18,291 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:18,296 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:18,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:19,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:23,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:23,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:23,715 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:23,716 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:23,721 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:24,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:24,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:14:25,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:28,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:29,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:29,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:29,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:29,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:29,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:29,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:29,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:29,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:31,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:32,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:32,639 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:32,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:32,644 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:33,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:34,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:35,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:36,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:36,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:36,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:36,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:37,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:37,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:40,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:40,748 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:40,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:40,749 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:40,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:41,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:43,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:44,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:44,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:44,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:46,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:46,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:46,414 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:46,415 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:46,419 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:47,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:47,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:51,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:51,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:51,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:51,247 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:51,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:52,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:53,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:54,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:14:55,810 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:55,811 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:55,811 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:55,816 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:56,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:57,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:57,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:59,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:59,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:59,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:59,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:59,819 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:59,819 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:59,824 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:00,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:01,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:02,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:04,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:04,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:04,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:04,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:04,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:05,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:08,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:08,142 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:08,143 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:08,143 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:08,148 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:08,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:09,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:13,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:13,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:13,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:13,496 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:13,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:14,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:14,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:14,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:14,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:15,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:17,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:17,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:17,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:17,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:18,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:18,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:19,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:21,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:21,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:21,626 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:21,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:22,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:23,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:24,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:24,752 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:15:27,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:27,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:27,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:27,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:27,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:29,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:29,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:29,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:29,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:30,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:31,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:31,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:31,234 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:31,239 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:31,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:33,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:35,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:35,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:35,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:35,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:35,693 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:36,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:37,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:39,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:39,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:39,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:39,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:40,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:40,759 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:41,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:43,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:43,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:43,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:43,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:43,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:44,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:45,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:46,526 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:50,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:50,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:50,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:50,105 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:50,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:51,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:52,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:54,454 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:54,455 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:54,455 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:54,460 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:54,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:15:55,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:55,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:57,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:58,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:58,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:58,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:58,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:59,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:59,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:59,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:59,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:01,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:02,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:04,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:04,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:04,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:04,121 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:04,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:05,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:08,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:09,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:09,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:09,835 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:09,840 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:10,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:11,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:13,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:14,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:14,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:14,647 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:14,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:14,648 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:14,653 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:15,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:15,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:19,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:20,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:20,339 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:20,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:20,344 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:20,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:21,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:24,523 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:24,524 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:24,524 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:24,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:24,756 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:16:24,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:25,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:25,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:29,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:29,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:29,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:29,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:29,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:29,531 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:29,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:30,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:30,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:31,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:33,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:33,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:33,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:33,799 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:34,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:35,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:35,800 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:40,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:40,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:40,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:40,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:40,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:41,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:41,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:44,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:44,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:46,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:47,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:47,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:47,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:47,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:47,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:49,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:51,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:51,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:51,167 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:51,172 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:51,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:52,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:53,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:54,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:16:55,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:55,350 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:55,351 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:55,356 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:55,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:57,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:57,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:59,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:59,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:59,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:59,870 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:59,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:59,871 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:59,875 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:00,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:01,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:02,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:06,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:06,400 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:06,400 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:06,405 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:06,489 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:07,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:08,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:10,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:10,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:10,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:10,119 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:10,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:11,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:14,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:14,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:14,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:14,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:14,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:14,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:14,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:14,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:15,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:15,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:19,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:20,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:20,713 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:20,714 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:20,719 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:21,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:21,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:24,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:17:25,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:25,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:25,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:25,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:25,190 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:25,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:27,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:29,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:29,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:29,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:29,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:29,704 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:29,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:29,709 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:30,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:30,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:31,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:35,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:36,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:36,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:36,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:36,340 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:36,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:37,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:40,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:40,933 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:40,933 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:40,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:40,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:41,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:43,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:44,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:44,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:44,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:44,743 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:44,743 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:44,748 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:45,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:45,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:46,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:49,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:49,068 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:49,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:49,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:49,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:51,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:52,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:54,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:54,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:54,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:54,078 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:54,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:54,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:17:55,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:57,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:57,768 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:57,768 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:57,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:57,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:58,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:59,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:59,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:59,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:59,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:01,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:01,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:01,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:01,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:02,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:02,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:03,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:05,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:05,473 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:05,474 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:05,479 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:05,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:07,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:08,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:10,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:10,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:10,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:10,452 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:10,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:11,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:14,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:14,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:14,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:14,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:14,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:14,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:14,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:14,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:15,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:15,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:18,760 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:18,761 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:18,761 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:18,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:19,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:19,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:19,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:22,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:22,506 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:22,506 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:22,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:22,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:23,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:24,768 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:18:24,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:26,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:26,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:26,865 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:26,870 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:27,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:27,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:29,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:29,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:29,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:30,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:32,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:32,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:32,380 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:32,385 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:32,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:33,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:35,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:35,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:35,923 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:35,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:35,928 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:36,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:37,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:40,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:44,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:44,119 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:44,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:44,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:44,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:44,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:45,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:46,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:47,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:47,826 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:47,826 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:47,831 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:48,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:49,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:51,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:52,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:52,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:52,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:52,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:52,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:53,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:54,771 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:18:55,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:55,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:55,824 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:55,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:56,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:57,607 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:57,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:59,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:59,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:59,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:59,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:59,586 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:59,586 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:59,591 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:59,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:01,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:03,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:04,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:04,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:04,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:04,999 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:05,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:07,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:09,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:09,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:09,563 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:09,563 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:09,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:09,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:11,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:13,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:13,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:13,556 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:13,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:13,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:14,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:14,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:14,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:15,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:17,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:17,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:17,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:17,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:17,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:19,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:20,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:21,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:21,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:21,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:21,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:21,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:23,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:24,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:19:25,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:25,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:25,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:25,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:25,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:26,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:27,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:29,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:29,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:29,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:31,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:34,430 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:34,431 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:34,431 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:34,436 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:34,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:35,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:37,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:39,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:39,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:39,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:39,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:39,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:41,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:43,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:44,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:44,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:44,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:44,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:44,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:44,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:45,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:48,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:48,359 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:48,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:48,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:48,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:48,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:49,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:53,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:53,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:53,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:53,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:53,444 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:53,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:54,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:19:55,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:58,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:59,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:59,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:59,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:00,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:00,480 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:00,480 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:00,485 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:00,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:01,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:04,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:05,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:05,091 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:05,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:05,096 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:05,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:07,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:09,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:09,101 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:09,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:09,106 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:09,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:10,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:11,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:13,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:13,359 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:13,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:13,365 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:13,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:14,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:14,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:15,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:15,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:19,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:19,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:19,506 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:19,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:19,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:21,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:21,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:24,780 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:20:26,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:27,868 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:27,868 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:27,869 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:27,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:28,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:29,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:29,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:29,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:29,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:31,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:31,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:31,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:31,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:31,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:32,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:33,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:35,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:35,900 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:35,900 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:35,905 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:36,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:37,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:37,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:40,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:40,057 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:40,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:40,063 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:40,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:41,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:43,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:44,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:44,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:44,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:48,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:49,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:49,265 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:49,265 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:49,270 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:49,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:51,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:53,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:53,276 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:53,276 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:53,281 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:53,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:54,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:54,783 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:20:55,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:58,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:58,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:58,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:58,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:58,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:59,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:59,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:59,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:59,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:59,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:02,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:02,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:02,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:02,761 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:03,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:03,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:04,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:07,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:07,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:07,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:07,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:07,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:09,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:10,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:12,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:12,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:12,542 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:12,547 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:12,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:13,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:14,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:15,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:16,426 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:16,426 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:16,426 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:16,432 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:16,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:17,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:21,469 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:22,762 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:22,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:22,763 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:22,768 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:23,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:23,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:24,789 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:21:26,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:26,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:26,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:26,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:26,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:27,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:27,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:29,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:29,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:29,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:30,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:30,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:30,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:30,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:30,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:31,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:32,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:34,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:34,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:34,363 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:34,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:34,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:35,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:38,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:38,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:38,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:38,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:38,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:38,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:39,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:43,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:44,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:44,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:44,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:45,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:45,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:45,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:45,785 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:45,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:47,789 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:48,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:50,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:50,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:50,192 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:50,197 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:50,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:51,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:53,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:53,587 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:53,587 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:53,592 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:53,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:54,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:54,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:21:55,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:59,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:59,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:59,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:00,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:02,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:02,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:02,555 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:02,560 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:02,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:03,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:05,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:06,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:06,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:06,279 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:06,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:06,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:07,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:11,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:11,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:11,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:11,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:11,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:11,810 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:13,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:14,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:14,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:16,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:16,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:16,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:16,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:16,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:17,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:17,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:21,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:21,503 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:21,503 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:21,508 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:21,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:22,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:23,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:24,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:22:27,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:29,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:29,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:29,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:29,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:29,902 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:29,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:29,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:30,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:31,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:32,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:33,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:33,425 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:33,425 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:33,430 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:33,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:35,836 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:38,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:38,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:38,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:38,432 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:38,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:38,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:39,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:41,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:41,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:41,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:41,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:42,842 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:43,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:43,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:44,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:44,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:44,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:49,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:50,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:50,277 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:50,277 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:50,295 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:50,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:51,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:54,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:22:54,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:55,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:55,551 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:55,551 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:55,556 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:55,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:57,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:59,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:59,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:59,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:00,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:01,430 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:01,431 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:01,431 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:01,436 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:01,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:03,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:06,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:07,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:07,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:07,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:07,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:07,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:09,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:11,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:11,600 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:11,600 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:11,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:11,605 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:11,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:13,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:14,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:14,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:15,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:15,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:15,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:15,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:16,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:16,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:17,880 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:20,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:20,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:20,283 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:20,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:20,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:21,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:22,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:24,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:24,691 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:24,691 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:24,696 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:24,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:23:24,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:25,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:27,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:29,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:29,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:29,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:29,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:29,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:29,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:29,531 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:29,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:31,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:32,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:32,934 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:32,934 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:32,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:32,939 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:33,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:33,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:37,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:37,392 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:37,392 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:37,397 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:37,902 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:38,398 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:39,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:41,803 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:41,803 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:41,804 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:41,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:41,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:43,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:43,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:44,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:44,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:44,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:45,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:45,533 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:45,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:45,538 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:45,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:47,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:49,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:49,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:49,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:49,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:49,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:49,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:51,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:54,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:54,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:54,135 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:54,140 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:54,798 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:23:54,799 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:54,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:55,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:59,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:59,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:59,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:00,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:00,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:00,198 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:00,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:00,203 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:00,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:01,927 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:05,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:08,483 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:08,484 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:08,484 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:08,489 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:08,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:09,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:10,490 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:14,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:14,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:14,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:15,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:17,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:17,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:17,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:17,298 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:17,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:17,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:21,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:21,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:21,367 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:21,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:21,372 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:21,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:21,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:24,801 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:24:25,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:25,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:25,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:25,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:25,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:25,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:27,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:29,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:29,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:29,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:30,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:30,203 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:30,204 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:30,208 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:30,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:31,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:32,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:36,746 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:36,747 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:36,747 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:36,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:36,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:37,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:37,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:40,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:40,523 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:40,523 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:40,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:40,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:41,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:43,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:44,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:44,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:44,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:45,106 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:45,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:45,111 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:45,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:45,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:49,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:49,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:49,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:49,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:49,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:49,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:51,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:54,807 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:24:54,808 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:54,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:54,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:54,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:54,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:54,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:55,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:59,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:59,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:00,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:03,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:03,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:03,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:03,325 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:03,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:03,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:06,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:07,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:07,295 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:07,295 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:07,300 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:07,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:07,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:11,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:11,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:11,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:11,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:12,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:12,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:12,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:14,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:14,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:14,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:16,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:16,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:16,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:16,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:17,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:17,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:18,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:20,469 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:20,470 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:20,470 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:20,475 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:21,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:22,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:23,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:24,807 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:25:28,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:28,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:28,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:28,861 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:28,866 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:29,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:29,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:29,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:29,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:30,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:33,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:33,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:33,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:33,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:34,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:34,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:34,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:37,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:37,132 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:37,132 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:37,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:38,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:38,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:40,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:44,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:44,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:44,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:44,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:44,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:44,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:44,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:45,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:45,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:46,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:48,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:48,789 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:48,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:48,794 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:49,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:50,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:51,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:52,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:52,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:52,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:52,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:53,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:54,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:54,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:25:56,815 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:57,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:57,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:57,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:57,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:58,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:58,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:59,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:01,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:01,780 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:01,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:01,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:01,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:02,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:04,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:06,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:06,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:06,214 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:06,219 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:07,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:07,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:08,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:11,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:11,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:11,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:11,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:12,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:12,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:13,179 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:14,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:14,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:15,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:15,153 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:15,153 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:15,158 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:16,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:16,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:18,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:21,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:21,025 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:21,025 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:21,030 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:21,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:22,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:24,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:24,812 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:26:24,863 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:24,864 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:24,864 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:24,869 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:25,075 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:26,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:28,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:28,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:28,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:28,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:29,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:29,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:29,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:29,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:29,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:30,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:32,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:32,429 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:32,429 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:32,434 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:33,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:34,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:35,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:37,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:37,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:37,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:37,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:38,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:38,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:41,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:41,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:41,620 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:41,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:41,625 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:42,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:44,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:44,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:44,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:44,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:45,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:45,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:45,678 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:45,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:46,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:46,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:48,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:50,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:50,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:50,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:50,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:51,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:52,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:52,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:54,816 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:26:55,764 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:55,768 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:55,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:55,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:56,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:57,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:58,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:59,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:59,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:59,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:59,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:59,760 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:00,113 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:02,116 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:02,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:04,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:04,429 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:04,429 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:04,434 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:05,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:06,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:08,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:10,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:10,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:10,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:10,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:11,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:12,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:14,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:14,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:14,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:15,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:15,616 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:15,616 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:15,621 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:16,128 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:18,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:19,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:20,824 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:20,825 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:20,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:20,831 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:21,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:22,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:24,818 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:27:24,820 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:25,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:25,411 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:25,411 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:25,416 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:26,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:26,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:29,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:29,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:29,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:30,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:30,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:30,039 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:30,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:30,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:30,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:32,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:35,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:38,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:38,346 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:38,346 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:38,351 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:39,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:40,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:40,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:42,982 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:42,983 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:42,983 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:42,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:43,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:44,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:44,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:44,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:44,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:45,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:49,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:49,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:49,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:49,557 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:50,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:50,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:51,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:53,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:53,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:53,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:53,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:54,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:54,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:54,820 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:27:56,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:57,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:57,898 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:57,898 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:57,903 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:58,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:59,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:59,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:00,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:01,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:01,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:01,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:01,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:01,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:02,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:04,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:06,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:09,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:09,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:09,854 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:09,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:10,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:12,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:12,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:13,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:13,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:13,997 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:14,003 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:14,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:14,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:16,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:18,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:18,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:18,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:18,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:18,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:19,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:20,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:23,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:23,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:23,846 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:23,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:23,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:24,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:24,826 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:28:26,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:28,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:28,543 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:28,543 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:28,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:28,548 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:29,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:29,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:29,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:30,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:32,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:32,558 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:32,558 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:32,563 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:33,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:33,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:34,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:38,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:38,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:38,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:38,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:39,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:39,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:40,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:42,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:42,866 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:42,866 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:42,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:43,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:44,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:44,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:44,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:44,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:45,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:46,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:46,641 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:46,641 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:46,646 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:47,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:48,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:50,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:51,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:51,178 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:51,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:51,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:51,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:52,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:54,826 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:28:55,827 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:59,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:59,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:59,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:59,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:59,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:59,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:59,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:00,232 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:00,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:02,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:03,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:03,985 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:03,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:03,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:04,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:05,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:06,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:07,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:07,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:07,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:07,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:08,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:08,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:11,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:12,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:12,177 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:12,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:12,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:12,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:14,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:14,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:14,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:16,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:16,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:16,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:16,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:16,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:17,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:18,253 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:20,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:20,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:20,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:20,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:20,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:22,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:23,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:23,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:23,945 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:23,945 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:23,950 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:24,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:24,834 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:29:26,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:28,835 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:29,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:29,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:32,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:32,466 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:32,466 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:32,471 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:33,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:34,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:34,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:35,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:35,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:35,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:35,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:36,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:38,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:39,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:40,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:40,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:40,692 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:40,697 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:41,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:42,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:44,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:44,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:44,798 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:44,799 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:44,799 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:44,804 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:45,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:45,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:46,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:48,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:48,487 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:48,487 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:48,492 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:49,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:50,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:51,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:52,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:52,734 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:52,735 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:52,740 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:53,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:54,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:54,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:29:56,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:56,563 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:56,563 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:56,563 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:56,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:57,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:58,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:59,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:59,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:01,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:01,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:01,394 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:01,399 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:02,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:02,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:02,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:04,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:04,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:04,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:04,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:05,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:06,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:07,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:09,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:09,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:09,572 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:09,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:10,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:10,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:13,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:14,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:16,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:16,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:16,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:16,627 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:17,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:18,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:18,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:20,792 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:20,793 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:20,793 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:20,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:21,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:22,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:23,799 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:24,837 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:30:28,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:29,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:29,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:29,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:29,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:29,785 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:29,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:29,791 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:30,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:32,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:33,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:33,344 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:33,344 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:33,349 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:34,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:34,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:36,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:37,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:37,041 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:37,041 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:37,046 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:37,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:38,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:40,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:40,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:40,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:40,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:40,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:41,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:42,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:44,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:44,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:45,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:45,485 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:45,485 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:45,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:45,490 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:46,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:48,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:49,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:49,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:49,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:49,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:49,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:50,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:51,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:54,284 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:54,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:54,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:54,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:54,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:54,836 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:30:56,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:56,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:58,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:58,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:58,554 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:58,559 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:59,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:59,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:59,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:00,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:02,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:06,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:06,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:06,899 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:06,904 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:07,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:07,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:08,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:10,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:10,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:10,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:10,814 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:11,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:12,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:13,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:14,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:16,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:16,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:16,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:16,744 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:17,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:18,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:19,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:20,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:20,694 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:20,695 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:20,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:21,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:22,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:24,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:31:24,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:25,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:25,448 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:25,448 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:25,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:26,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:28,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:29,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:29,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:29,889 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:29,889 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:29,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:29,894 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:30,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:32,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:34,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:37,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:37,025 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:37,025 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:37,030 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:37,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:38,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:40,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:40,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:40,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:40,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:40,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:41,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:42,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:44,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:44,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:44,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:44,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:44,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:44,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:45,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:45,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:46,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:50,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:53,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:53,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:53,144 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:53,149 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:53,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:54,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:54,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:31:55,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:58,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:58,789 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:58,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:58,794 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:59,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:59,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:59,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:00,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:01,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:03,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:03,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:03,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:03,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:03,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:06,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:07,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:08,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:08,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:08,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:08,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:09,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:10,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:12,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:14,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:14,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:15,423 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:15,424 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:15,424 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:15,429 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:16,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:18,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:18,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:20,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:20,940 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:20,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:20,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:21,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:22,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:23,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:24,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:24,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:24,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:24,440 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:24,843 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:32:25,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:26,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:29,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:29,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:32,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:32,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:32,835 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:32,840 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:33,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:34,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:34,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:36,762 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:36,763 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:36,763 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:36,769 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:37,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:38,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:40,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:41,928 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:41,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:41,929 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:41,934 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:42,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:44,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:44,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:44,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:46,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:46,396 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:46,396 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:46,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:46,401 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:46,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:48,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:50,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:50,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:50,968 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:50,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:51,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:51,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:52,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:54,850 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:32:55,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:55,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:55,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:55,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:56,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:57,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:58,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:59,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:59,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:03,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:04,236 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:04,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:04,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:04,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:04,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:06,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:09,281 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:12,628 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:12,629 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:12,629 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:12,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:13,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:14,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:14,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:14,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:14,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:19,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:20,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:20,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:20,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:20,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:21,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:22,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:24,850 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:33:24,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:25,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:25,062 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:25,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:25,067 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:25,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:26,489 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:28,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:28,837 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:28,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:28,842 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:29,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:29,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:30,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:30,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:35,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:35,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:35,029 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:35,034 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:35,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:36,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:36,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:39,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:39,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:39,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:39,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:39,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:40,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:41,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:44,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:44,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:44,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:44,558 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:44,558 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:44,563 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:45,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:46,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:46,563 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:47,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:47,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:47,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:47,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:48,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:50,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:51,984 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:52,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:52,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:52,250 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:52,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:52,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:54,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:54,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:33:55,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:55,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:55,774 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:55,779 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:56,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:57,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:58,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:59,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:59,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:59,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:59,945 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:59,945 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:59,950 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:00,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:02,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:02,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:04,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:04,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:04,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:04,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:04,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:06,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:08,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:09,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:09,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:09,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:09,468 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:09,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:10,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:13,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:13,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:13,304 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:13,309 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:13,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:14,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:14,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:14,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:19,016 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:19,016 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:19,016 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:19,022 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:19,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:20,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:20,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:23,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:23,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:23,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:23,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:23,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:24,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:24,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:34:25,857 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:27,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:27,343 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:27,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:27,348 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:27,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:28,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:29,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:29,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:31,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:31,364 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:31,364 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:31,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:31,369 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:31,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:32,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:35,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:35,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:35,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:35,202 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:35,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:36,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:37,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:39,469 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:39,470 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:39,470 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:39,475 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:39,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:40,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:42,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:44,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:47,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:47,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:47,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:47,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:47,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:48,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:50,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:51,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:51,701 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:51,702 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:51,707 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:52,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:52,707 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:54,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:54,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:34:57,236 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:57,237 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:57,237 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:57,251 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:57,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:58,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:58,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:59,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:02,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:02,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:02,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:02,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:03,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:03,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:04,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:08,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:10,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:10,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:10,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:10,144 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:10,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:12,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:14,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:14,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:15,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:15,203 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:15,203 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:15,208 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:15,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:16,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:18,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:18,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:18,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:18,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:19,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:19,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:20,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:22,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:22,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:22,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:22,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:23,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:24,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:24,861 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:35:25,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:29,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:29,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:31,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:31,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:31,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:31,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:31,252 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:31,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:32,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:35,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:35,196 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:35,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:35,202 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:35,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:36,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:37,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:41,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:41,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:41,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:41,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:41,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:42,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:42,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:44,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:45,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:45,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:45,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:45,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:46,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:47,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:48,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:50,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:50,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:50,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:50,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:51,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:52,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:53,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:54,864 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:35:55,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:55,414 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:55,414 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:55,419 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:55,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:56,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:59,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:59,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:00,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:00,388 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:00,388 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:00,393 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:00,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:02,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:04,526 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:04,527 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:04,527 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:04,532 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:04,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:05,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:06,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:09,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:09,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:09,616 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:09,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:09,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:10,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:10,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:13,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:13,630 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:13,630 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:13,635 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:13,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:14,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:14,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:16,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:21,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:22,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:22,064 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:22,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:22,069 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:22,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:24,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:24,871 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:36:25,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:25,933 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:25,933 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:25,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:26,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:26,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:28,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:29,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:29,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:29,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:29,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:29,989 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:29,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:30,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:32,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:32,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:34,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:34,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:34,627 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:34,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:34,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:36,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:37,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:39,908 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:39,909 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:39,909 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:39,914 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:40,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:42,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:42,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:43,871 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:43,872 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:43,873 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:43,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:44,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:44,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:46,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:47,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:47,901 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:47,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:47,907 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:47,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:48,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:50,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:52,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:52,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:52,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:52,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:52,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:53,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:54,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:54,871 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:36:56,668 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:56,668 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:56,669 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:56,673 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:56,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:58,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:58,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:59,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:01,231 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:01,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:01,232 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:01,237 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:01,708 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:02,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:04,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:06,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:06,165 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:06,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:06,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:06,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:08,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:10,171 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:14,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:15,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:15,895 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:15,896 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:15,896 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:15,900 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:16,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:18,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:20,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:20,262 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:20,262 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:20,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:20,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:21,267 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:22,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:24,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:24,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:24,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:24,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:24,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:24,876 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:37:26,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:26,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:28,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:28,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:28,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:28,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:28,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:29,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:30,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:32,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:34,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:34,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:34,696 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:34,701 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:34,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:36,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:37,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:38,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:38,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:38,624 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:38,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:38,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:40,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:43,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:43,576 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:43,577 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:43,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:43,583 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:43,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:44,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:44,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:48,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:49,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:49,010 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:49,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:49,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:49,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:50,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:54,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:54,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:54,097 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:54,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:54,102 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:54,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:54,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:37:56,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:58,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:58,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:58,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:58,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:58,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:59,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:59,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:00,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:02,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:02,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:02,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:02,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:02,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:04,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:04,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:07,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:07,322 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:07,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:07,328 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:07,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:08,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:09,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:11,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:11,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:11,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:11,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:11,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:12,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:14,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:14,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:14,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:16,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:16,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:16,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:16,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:16,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:18,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:19,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:23,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:23,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:23,496 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:23,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:23,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:24,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:24,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:38:24,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:27,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:27,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:27,660 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:27,664 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:27,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:28,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:29,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:29,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:30,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:32,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:32,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:32,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:32,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:33,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:34,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:35,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:37,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:37,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:37,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:37,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:37,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:38,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:41,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:43,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:43,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:43,136 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:43,141 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:43,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:44,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:44,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:47,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:47,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:47,589 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:47,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:47,594 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:47,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:48,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:52,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:53,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:53,290 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:53,290 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:53,295 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:53,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:54,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:54,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:38:57,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:58,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:58,170 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:58,171 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:58,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:58,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:59,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:59,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:00,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:02,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:02,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:02,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:02,340 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:02,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:03,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:04,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:06,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:06,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:06,222 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:06,227 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:06,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:08,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:09,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:12,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:12,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:12,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:12,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:12,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:14,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:14,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:14,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:14,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:14,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:16,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:16,145 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:16,145 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:16,150 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:16,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:18,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:20,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:21,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:21,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:21,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:21,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:21,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:22,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:24,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:39:25,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:25,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:25,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:25,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:25,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:26,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:26,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:29,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:29,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:29,250 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:29,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:29,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:29,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:29,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:29,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:30,867 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:31,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:32,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:32,595 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:32,595 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:32,600 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:32,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:34,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:36,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:37,468 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:37,469 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:37,469 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:37,474 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:37,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:38,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:42,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:44,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:44,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:44,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:45,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:45,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:45,282 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:45,287 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:45,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:46,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:48,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:49,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:49,584 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:49,584 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:49,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:49,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:50,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:53,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:53,506 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:53,506 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:53,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:53,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:53,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:54,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:39:54,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:57,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:57,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:57,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:57,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:58,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:58,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:58,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:59,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:59,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:59,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:02,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:02,061 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:02,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:02,066 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:02,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:04,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:04,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:06,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:06,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:06,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:06,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:06,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:08,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:10,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:14,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:14,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:14,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:15,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:17,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:17,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:17,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:17,043 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:17,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:18,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:20,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:20,861 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:20,861 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:20,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:20,866 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:20,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:22,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:24,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:40:25,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:26,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:26,389 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:26,389 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:26,394 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:26,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:28,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:29,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:29,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:29,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:31,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:32,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:32,050 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:32,050 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:32,055 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:32,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:34,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:37,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:37,070 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:37,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:37,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:37,075 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:37,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:38,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:42,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:42,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:42,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:42,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:42,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:42,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:44,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:44,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:44,942 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:47,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:47,998 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:47,999 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:47,999 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:48,004 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:48,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:50,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:51,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:51,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:51,942 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:51,951 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:51,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:52,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:52,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:54,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:40:57,250 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:57,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:57,251 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:57,256 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:57,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:58,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:58,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:59,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:59,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:59,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:01,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:01,452 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:01,452 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:01,457 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:01,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:02,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:03,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:06,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:06,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:06,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:06,251 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:06,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:08,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:09,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:10,986 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:10,987 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:10,987 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:10,992 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:11,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:12,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:14,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:14,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:14,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:14,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:16,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:16,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:16,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:16,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:16,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:18,979 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:20,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:24,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:24,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:24,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:24,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:24,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:41:24,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:25,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:26,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:28,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:28,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:28,870 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:28,875 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:28,987 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:29,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:29,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:30,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:31,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:36,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:37,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:37,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:37,805 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:37,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:37,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:38,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:41,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:42,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:42,595 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:42,595 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:42,600 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:43,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:44,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:44,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:45,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:47,236 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:47,237 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:47,237 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:47,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:47,242 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:48,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:49,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:51,442 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:51,442 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:51,443 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:51,447 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:52,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:52,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:53,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:54,897 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:41:56,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:56,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:56,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:56,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:57,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:58,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:59,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:59,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:59,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:59,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:03,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:04,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:04,269 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:04,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:04,274 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:05,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:07,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:08,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:08,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:08,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:08,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:08,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:09,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:11,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:13,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:14,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:14,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:14,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:17,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:17,022 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:17,022 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:17,027 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:17,033 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:19,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:19,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:21,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:21,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:21,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:21,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:22,037 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:23,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:24,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:24,896 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:42:24,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:24,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:24,992 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:24,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:25,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:27,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:29,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:29,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:29,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:31,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:31,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:31,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:31,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:32,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:33,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:35,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:35,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:35,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:35,289 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:35,293 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:36,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:37,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:40,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:40,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:40,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:40,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:40,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:41,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:43,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:44,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:44,464 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:44,596 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:44,596 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:44,601 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:45,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:45,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:47,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:48,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:48,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:48,683 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:48,688 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:49,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:50,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:51,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:52,450 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:52,451 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:52,451 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:52,456 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:53,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:54,897 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:42:55,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:55,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:58,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:58,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:58,835 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:58,840 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:59,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:59,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:59,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:59,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:01,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:01,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:02,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:02,792 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:02,792 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:02,796 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:03,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:05,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:06,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:06,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:06,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:06,954 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:06,959 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:07,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:09,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:10,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:10,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:10,719 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:10,724 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:11,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:12,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:13,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:14,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:14,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:14,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:15,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:15,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:15,303 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:15,308 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:16,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:17,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:18,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:18,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:18,972 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:18,972 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:18,977 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:19,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:21,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:23,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:23,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:23,387 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:23,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:23,392 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:24,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:24,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:43:25,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:27,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:27,331 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:27,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:27,337 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:28,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:29,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:29,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:29,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:32,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:32,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:32,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:32,580 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:33,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:34,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:35,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:36,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:36,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:36,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:36,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:37,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:37,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:39,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:39,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:39,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:39,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:39,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:40,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:41,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:43,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:43,529 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:43,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:43,534 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:44,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:44,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:44,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:45,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:45,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:47,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:47,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:47,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:47,504 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:48,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:49,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:51,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:51,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:51,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:51,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:51,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:52,128 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:53,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:54,902 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:43:55,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:55,397 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:55,398 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:55,403 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:56,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:56,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:57,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:59,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:59,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:59,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:59,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:59,837 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:59,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:59,842 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:00,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:01,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:01,842 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:04,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:04,573 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:04,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:04,578 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:05,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:07,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:07,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:08,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:08,131 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:08,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:08,136 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:08,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:09,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:13,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:14,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:14,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:14,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:14,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:14,611 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:14,611 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:14,616 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:15,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:17,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:18,392 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:18,393 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:18,393 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:18,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:18,398 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:19,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:21,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:22,815 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:22,816 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:22,816 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:22,821 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:23,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:23,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:24,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:44:25,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:27,236 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:27,237 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:27,237 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:27,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:28,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:29,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:29,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:29,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:31,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:31,194 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:31,194 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:31,198 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:32,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:33,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:35,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:35,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:35,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:35,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:35,660 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:36,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:37,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:39,188 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:39,189 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:39,189 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:39,194 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:39,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:41,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:41,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:44,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:44,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:44,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:44,247 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:44,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:44,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:44,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:45,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:45,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:46,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:48,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:48,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:48,566 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:48,571 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:49,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:51,212 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:51,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:54,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:44:55,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:55,931 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:55,931 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:55,936 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:56,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:56,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:57,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:59,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:59,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:59,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:00,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:00,159 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:00,160 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:00,165 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:00,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:01,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:02,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:04,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:04,464 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:04,464 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:04,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:05,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:07,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:07,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:10,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:10,286 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:10,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:10,291 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:11,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:13,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:13,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:14,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:14,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:14,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:14,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:14,690 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:14,690 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:14,695 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:15,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:17,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:18,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:18,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:18,918 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:18,918 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:18,923 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:19,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:21,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:22,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:22,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:22,925 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:22,930 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:23,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:23,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:24,911 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:45:25,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:28,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:28,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:28,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:28,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:29,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:29,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:29,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:29,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:31,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:33,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:33,902 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:33,902 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:33,907 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:34,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:34,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:35,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:37,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:37,771 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:37,771 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:37,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:38,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:39,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:40,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:43,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:43,178 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:43,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:43,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:43,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:44,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:44,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:45,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:46,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:51,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:51,441 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:51,441 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:51,446 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:52,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:52,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:53,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:54,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:45:55,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:55,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:55,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:55,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:56,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:57,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:57,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:59,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:59,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:59,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:59,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:59,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:59,617 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:59,622 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:00,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:01,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:03,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:03,762 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:03,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:03,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:03,768 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:04,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:05,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:08,231 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:08,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:08,232 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:08,238 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:08,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:09,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:09,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:12,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:12,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:12,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:12,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:13,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:14,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:14,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:14,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:14,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:15,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:16,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:16,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:16,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:16,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:17,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:19,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:19,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:21,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:21,635 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:21,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:21,641 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:22,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:23,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:24,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:46:24,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:27,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:27,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:27,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:27,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:27,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:29,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:29,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:29,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:30,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:31,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:31,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:31,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:31,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:31,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:33,361 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:35,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:35,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:35,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:35,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:35,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:36,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:37,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:38,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:38,400 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:38,400 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:38,405 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:39,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:41,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:41,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:42,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:42,127 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:42,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:42,132 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:42,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:43,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:44,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:46,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:46,799 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:46,800 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:46,800 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:46,805 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:47,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:49,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:50,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:50,939 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:50,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:50,944 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:51,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:51,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:53,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:54,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:54,602 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:54,602 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:54,607 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:54,914 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:46:55,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:57,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:57,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:59,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:59,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:59,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:59,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:59,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:59,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:59,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:00,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:01,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:03,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:05,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:05,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:05,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:05,729 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:06,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:07,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:08,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:10,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:10,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:10,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:10,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:11,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:13,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:13,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:14,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:14,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:14,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:16,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:16,066 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:16,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:16,071 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:16,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:17,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:19,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:21,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:21,155 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:21,156 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:21,161 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:21,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:23,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:24,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:24,921 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:47:26,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:26,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:26,148 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:26,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:26,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:27,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:29,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:29,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:29,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:29,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:29,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:29,897 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:29,897 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:29,902 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:30,447 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:31,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:34,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:34,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:34,014 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:34,018 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:34,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:35,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:35,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:40,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:42,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:42,636 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:42,636 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:42,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:43,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:44,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:45,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:45,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:47,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:47,576 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:47,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:47,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:48,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:49,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:51,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:51,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:51,639 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:51,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:51,644 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:52,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:53,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:54,918 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:47:55,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:55,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:55,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:55,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:56,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:56,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:57,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:59,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:59,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:01,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:01,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:01,830 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:01,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:02,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:02,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:03,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:06,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:06,177 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:06,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:06,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:06,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:07,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:08,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:11,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:11,717 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:11,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:11,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:12,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:13,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:13,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:14,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:14,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:14,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:17,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:17,172 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:17,172 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:17,177 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:17,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:19,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:19,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:21,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:21,563 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:21,563 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:21,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:22,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:23,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:24,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:24,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:48:26,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:26,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:26,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:26,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:27,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:29,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:29,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:29,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:29,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:30,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:35,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:35,357 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:35,357 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:35,362 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:35,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:36,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:37,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:39,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:39,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:39,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:39,173 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:39,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:41,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:42,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:43,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:43,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:43,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:43,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:44,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:44,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:45,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:47,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:50,700 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:50,701 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:50,701 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:50,706 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:51,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:52,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:53,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:54,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:48:56,835 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:56,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:56,836 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:56,840 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:57,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:57,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:59,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:59,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:59,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:01,928 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:01,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:01,928 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:01,933 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:02,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:02,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:03,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:07,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:07,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:07,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:07,549 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:07,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:08,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:09,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:11,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:11,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:11,039 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:11,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:11,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:13,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:14,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:14,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:14,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:14,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:16,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:16,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:16,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:16,356 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:16,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:17,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:19,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:20,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:20,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:20,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:20,879 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:21,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:23,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:24,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:24,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:49:26,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:26,262 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:26,262 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:26,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:26,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:27,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:29,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:29,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:29,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:30,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:31,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:31,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:31,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:31,365 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:31,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:33,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:35,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:35,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:35,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:35,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:35,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:36,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:37,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:39,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:39,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:39,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:39,339 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:39,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:41,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:42,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:44,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:44,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:45,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:45,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:45,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:45,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:46,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:47,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:47,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:52,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:54,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:54,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:54,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:54,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:54,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:54,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:49:55,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:57,608 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:57,609 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:57,609 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:57,614 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:57,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:58,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:59,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:59,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:59,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:01,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:01,938 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:01,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:01,943 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:02,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:03,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:03,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:06,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:06,371 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:06,371 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:06,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:06,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:07,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:09,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:11,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:11,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:11,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:11,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:11,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:13,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:14,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:14,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:14,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:14,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:15,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:15,585 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:15,585 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:15,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:15,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:17,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:19,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:20,800 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:20,801 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:20,801 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:20,806 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:21,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:23,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:24,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:24,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:50:25,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:25,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:25,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:25,644 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:25,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:27,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:29,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:29,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:29,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:29,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:29,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:29,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:29,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:29,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:30,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:31,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:33,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:33,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:33,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:33,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:34,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:35,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:35,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:38,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:38,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:38,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:38,328 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:38,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:39,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:41,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:43,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:43,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:43,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:43,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:43,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:44,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:44,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:45,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:46,616 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:47,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:47,308 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:47,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:47,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:47,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:49,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:52,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:53,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:53,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:53,003 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:53,008 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:53,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:54,933 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:50:55,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:56,759 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:56,760 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:56,760 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:56,765 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:57,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:57,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:59,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:59,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:59,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:59,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:00,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:00,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:00,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:00,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:01,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:02,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:03,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:05,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:05,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:05,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:05,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:06,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:07,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:08,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:09,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:09,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:09,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:09,795 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:10,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:11,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:13,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:13,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:13,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:13,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:13,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:14,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:14,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:14,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:14,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:15,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:19,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:20,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:20,161 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:20,161 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:20,166 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:20,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:21,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:23,995 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:23,996 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:23,996 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:24,001 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:24,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:24,932 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:51:24,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:25,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:27,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:27,436 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:27,436 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:27,441 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:27,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:29,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:29,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:29,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:29,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:30,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:31,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:31,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:31,659 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:31,664 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:31,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:33,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:35,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:37,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:37,571 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:37,571 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:37,576 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:37,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:39,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:41,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:41,329 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:41,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:41,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:41,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:41,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:43,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:44,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:44,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:44,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:45,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:45,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:45,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:45,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:45,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:46,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:47,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:49,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:49,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:49,464 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:49,468 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:49,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:51,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:52,469 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:53,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:53,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:53,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:53,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:53,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:54,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:51:55,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:57,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:59,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:59,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:59,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:59,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:59,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:59,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:59,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:59,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:00,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:01,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:03,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:03,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:03,640 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:03,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:03,645 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:03,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:05,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:08,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:09,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:09,430 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:09,430 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:09,435 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:09,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:11,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:13,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:13,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:13,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:13,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:13,779 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:13,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:14,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:14,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:14,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:15,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:17,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:17,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:17,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:17,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:17,789 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:19,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:19,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:21,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:21,558 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:21,558 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:21,564 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:21,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:23,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:24,936 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:52:24,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:29,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:29,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:29,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:29,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:29,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:29,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:29,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:29,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:30,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:31,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:32,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:32,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:32,919 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:32,924 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:33,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:35,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:35,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:39,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:39,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:39,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:39,201 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:39,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:41,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:41,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:44,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:44,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:44,827 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:44,828 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:44,828 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:44,833 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:45,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:46,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:47,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:49,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:49,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:49,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:49,692 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:49,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:51,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:52,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:54,938 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:52:55,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:55,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:55,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:55,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:56,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:57,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:57,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:59,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:59,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:59,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:59,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:59,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:59,802 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:59,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:01,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:03,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:04,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:04,331 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:04,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:04,336 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:04,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:05,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:08,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:08,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:08,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:08,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:08,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:09,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:09,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:14,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:14,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:14,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:14,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:16,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:16,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:16,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:16,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:16,849 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:17,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:19,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:21,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:21,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:21,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:21,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:21,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:23,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:24,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:24,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:53:26,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:26,254 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:26,254 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:26,259 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:26,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:27,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:29,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:29,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:29,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:30,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:32,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:32,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:32,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:32,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:32,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:33,867 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:36,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:37,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:37,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:37,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:37,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:37,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:39,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:41,142 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:41,143 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:41,143 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:41,148 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:41,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:42,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:43,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:44,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:45,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:45,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:45,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:45,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:45,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:47,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:47,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:50,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:50,010 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:50,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:50,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:50,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:51,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:53,016 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:54,426 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:54,427 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:54,427 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:54,432 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:54,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:54,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:53:55,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:58,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:59,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:59,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:59,320 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:59,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:59,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:59,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:01,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:04,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:07,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:07,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:07,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:07,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:07,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:09,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:09,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:12,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:12,117 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:12,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:12,122 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:12,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:13,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:14,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:14,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:14,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:15,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:17,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:17,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:17,666 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:17,671 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:17,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:19,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:20,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:21,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:21,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:21,505 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:21,510 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:21,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:23,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:24,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:54:25,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:25,673 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:25,673 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:25,678 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:25,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:26,679 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:27,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:29,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:29,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:29,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:31,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:31,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:31,209 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:31,214 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:31,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:32,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:33,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:37,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:37,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:37,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:37,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:37,584 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:37,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:39,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:42,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:43,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:43,988 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:43,988 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:43,993 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:44,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:44,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:45,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:48,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:49,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:49,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:49,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:49,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:49,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:51,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:54,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:54,084 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:54,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:54,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:54,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:54,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:54:54,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:55,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:58,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:58,077 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:58,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:58,082 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:58,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:59,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:59,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:59,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:59,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:02,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:02,568 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:02,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:02,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:02,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:03,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:04,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:08,015 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:08,016 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:08,016 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:08,021 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:08,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:09,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:10,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:12,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:12,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:12,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:12,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:12,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:13,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:14,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:14,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:14,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:15,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:16,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:16,672 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:16,672 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:16,677 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:16,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:17,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:20,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:20,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:20,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:20,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:20,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:20,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:21,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:24,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:55:25,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:26,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:26,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:26,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:26,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:26,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:27,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:29,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:29,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:29,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:31,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:32,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:32,067 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:32,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:32,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:32,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:33,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:36,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:36,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:36,298 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:36,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:36,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:37,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:37,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:40,452 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:40,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:40,453 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:40,458 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:40,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:41,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:42,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:44,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:44,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:44,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:44,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:44,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:44,724 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:45,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:46,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:47,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:49,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:49,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:49,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:49,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:50,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:52,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:52,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:52,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:52,860 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:52,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:52,864 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:53,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:54,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:54,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:55:57,016 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:57,017 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:57,017 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:57,022 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:58,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:58,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:59,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:59,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:00,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:01,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:01,218 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:01,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:01,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:02,021 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:03,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:04,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:04,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:04,968 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:04,968 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:04,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:05,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:06,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:08,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:08,815 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:08,815 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:08,815 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:08,820 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:09,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:10,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:13,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:13,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:13,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:13,795 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:14,035 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:14,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:14,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:14,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:14,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:16,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:19,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:19,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:19,279 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:19,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:20,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:20,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:22,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:23,171 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:23,172 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:23,172 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:23,177 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:24,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:24,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:56:25,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:26,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:28,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:28,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:28,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:28,391 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:29,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:29,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:29,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:29,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:30,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:31,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:32,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:32,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:32,373 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:32,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:33,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:34,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:37,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:41,824 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:41,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:41,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:41,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:42,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:42,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:44,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:44,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:44,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:44,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:47,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:47,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:47,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:47,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:47,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:48,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:50,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:51,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:51,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:51,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:51,444 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:52,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:53,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:54,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:54,955 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:56:55,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:55,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:55,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:55,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:56,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:58,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:58,760 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:59,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:59,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:01,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:01,576 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:01,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:01,580 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:02,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:04,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:04,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:05,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:05,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:05,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:05,580 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:06,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:08,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:10,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:13,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:13,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:13,951 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:13,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:14,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:14,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:14,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:14,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:16,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:16,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:17,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:17,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:17,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:17,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:18,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:20,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:21,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:21,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:21,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:21,778 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:21,783 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:22,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:24,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:24,957 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:57:25,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:25,914 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:25,914 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:25,919 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:26,106 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:26,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:28,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:29,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:29,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:29,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:29,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:29,763 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:29,763 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:29,768 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:30,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:32,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:32,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:35,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:35,128 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:35,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:35,133 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:36,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:38,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:38,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:40,751 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:40,752 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:40,752 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:40,757 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:41,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:42,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:43,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:44,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:44,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:44,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:46,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:46,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:46,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:46,292 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:47,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:48,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:49,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:52,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:52,964 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:52,965 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:52,969 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:53,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:54,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:54,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:57:54,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:57,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:57,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:57,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:57,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:58,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:59,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:59,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:00,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:00,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:02,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:02,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:02,526 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:02,531 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:03,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:04,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:06,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:08,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:08,833 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:08,834 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:08,839 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:09,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:10,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:11,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:12,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:12,809 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:12,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:12,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:13,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:14,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:14,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:14,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:14,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:16,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:16,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:16,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:16,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:17,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:17,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:18,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:20,482 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:20,482 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:20,483 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:20,488 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:21,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:22,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:23,488 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:24,961 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:58:24,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:24,992 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:24,992 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:24,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:25,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:26,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:28,998 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:29,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:29,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:30,101 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:30,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:30,106 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:30,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:32,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:34,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:35,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:35,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:35,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:35,298 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:36,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:38,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:39,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:39,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:39,532 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:39,532 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:39,538 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:40,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:42,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:43,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:43,451 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:43,452 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:43,457 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:44,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:44,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:44,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:46,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:47,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:47,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:47,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:47,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:48,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:49,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:50,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:51,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:51,846 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:51,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:51,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:52,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:54,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:54,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:54,966 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:58:55,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:55,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:55,974 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:55,979 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:56,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:58,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:59,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:59,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:59,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:59,852 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:59,852 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:59,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:59,857 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:00,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:02,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:04,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:04,178 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:04,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:04,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:04,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:05,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:06,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:08,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:08,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:08,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:08,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:09,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:10,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:10,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:12,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:12,551 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:12,551 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:12,557 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:13,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:14,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:14,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:14,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:14,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:15,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:17,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:17,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:17,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:17,247 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:18,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:20,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:21,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:22,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:22,494 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:22,494 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:22,499 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:23,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:24,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:24,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:59:26,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:26,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:26,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:26,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:26,444 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:27,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:28,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:29,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:29,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:30,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:30,477 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:30,477 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:30,482 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:31,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:31,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:32,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:35,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:35,076 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:35,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:35,081 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:35,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:36,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:37,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:42,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:43,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:43,223 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:43,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:43,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:43,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:44,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:44,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:44,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:47,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:48,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:48,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:48,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:48,090 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:48,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:50,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:53,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:53,478 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:53,479 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:53,479 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:53,484 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:54,263 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:54,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:59:56,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:58,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:59,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:59,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:59,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:59,454 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:59,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:59,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:59,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:00,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:02,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:03,986 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:03,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:03,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:03,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:03,992 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:04,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:06,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:08,868 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:08,869 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:08,869 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:08,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:09,280 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:09,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:10,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:14,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:14,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:14,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:14,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:14,947 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:14,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:14,953 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:15,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:16,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:18,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:18,478 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:18,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:18,483 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:19,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:20,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:20,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:24,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:24,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:24,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:24,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:24,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:24,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:00:25,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:26,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:28,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:28,740 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:28,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:28,746 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:29,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:29,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:29,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:30,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:31,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:32,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:32,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:32,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:32,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:33,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:34,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:36,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:37,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:37,163 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:37,164 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:37,169 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:37,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:38,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:42,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:42,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:42,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:42,916 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:42,921 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:43,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:44,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:44,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:44,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:44,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:47,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:48,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:48,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:48,370 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:48,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:49,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:50,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:52,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:52,630 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:52,631 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:52,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:52,636 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:53,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:54,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:54,972 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:00:56,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:56,957 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:56,957 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:56,962 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:57,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:57,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:58,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:59,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:02,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:02,625 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:02,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:02,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:03,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:03,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:04,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:07,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:07,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:07,989 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:07,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:08,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:08,995 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:10,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:12,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:12,958 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:12,958 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:12,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:13,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:14,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:14,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:14,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:14,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:14,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:16,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:16,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:16,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:16,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:17,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:18,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:19,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:22,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:22,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:22,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:22,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:22,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:24,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:24,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:01:24,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:25,904 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:25,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:25,905 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:25,910 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:26,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:28,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:29,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:29,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:29,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:29,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:29,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:29,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:29,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:29,361 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:30,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:30,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:34,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:34,073 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:34,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:34,078 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:34,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:36,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:36,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:40,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:40,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:40,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:40,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:41,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:41,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:42,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:44,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:44,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:44,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:44,775 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:44,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:44,780 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:45,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:46,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:46,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:48,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:48,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:48,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:48,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:48,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:50,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:51,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:51,756 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:51,757 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:51,762 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:52,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:52,762 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:54,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:54,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:01:55,850 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:55,850 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:55,851 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:55,856 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:56,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:57,856 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:58,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:59,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:59,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:59,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:59,560 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:59,560 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:59,565 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:00,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:02,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:03,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:08,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:08,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:08,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:08,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:09,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:09,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:10,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:13,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:13,434 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:13,434 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:13,439 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:14,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:14,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:14,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:15,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:16,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:16,976 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:16,977 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:16,977 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:16,982 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:17,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:18,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:21,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:21,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:21,586 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:21,586 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:21,592 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:22,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:24,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:24,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:02:25,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:25,549 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:25,550 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:25,555 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:26,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:26,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:28,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:29,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:29,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:29,303 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:29,308 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:29,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:29,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:29,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:30,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:32,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:33,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:33,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:33,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:33,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:34,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:36,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:37,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:38,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:38,037 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:38,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:38,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:38,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:40,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:42,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:42,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:42,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:42,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:42,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:43,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:44,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:44,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:44,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:47,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:47,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:47,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:47,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:48,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:48,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:50,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:51,727 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:51,728 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:51,728 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:51,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:52,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:53,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:54,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:54,981 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:02:58,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:58,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:58,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:58,191 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:58,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:59,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:59,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:59,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:59,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:00,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:02,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:02,071 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:02,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:02,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:02,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:04,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:05,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:05,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:05,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:05,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:05,886 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:06,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:08,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:10,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:13,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:13,866 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:13,867 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:13,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:14,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:14,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:16,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:16,538 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:18,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:18,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:18,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:18,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:18,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:20,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:22,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:24,987 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:03:26,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:26,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:26,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:26,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:27,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:27,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:28,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:29,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:31,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:31,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:31,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:31,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:31,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:32,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:33,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:37,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:37,502 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:37,502 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:37,507 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:38,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:38,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:38,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:40,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:40,934 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:40,934 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:40,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:41,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:42,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:43,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:44,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:44,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:45,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:45,961 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:45,962 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:45,967 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:46,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:48,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:49,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:51,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:51,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:51,018 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:51,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:51,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:52,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:54,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:54,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:54,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:54,805 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:54,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:54,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:03:55,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:56,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:59,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:59,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:59,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:59,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:01,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:01,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:01,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:01,377 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:01,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:02,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:05,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:06,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:06,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:06,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:06,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:07,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:08,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:10,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:10,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:10,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:10,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:10,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:11,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:12,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:14,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:16,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:16,219 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:16,219 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:16,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:16,224 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:16,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:18,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:21,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:24,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:24,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:24,731 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:24,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:24,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:04:25,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:26,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:26,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:28,976 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:28,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:28,977 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:28,982 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:29,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:29,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:29,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:30,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:32,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:33,752 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:33,753 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:33,753 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:33,758 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:34,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:36,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:37,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:37,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:37,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:37,928 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:37,933 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:38,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:40,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:42,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:43,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:43,130 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:43,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:43,136 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:43,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:44,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:44,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:44,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:48,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:48,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:48,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:48,672 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:48,677 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:49,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:50,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:52,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:52,635 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:52,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:52,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:53,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:53,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:54,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:54,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:04:56,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:56,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:56,404 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:56,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:56,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:58,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:59,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:59,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:59,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:59,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:00,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:00,272 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:00,272 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:00,277 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:00,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:02,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:05,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:05,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:05,867 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:05,867 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:05,873 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:06,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:08,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:10,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:10,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:10,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:10,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:10,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:11,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:12,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:14,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:14,102 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:14,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:14,107 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:14,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:14,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:14,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:16,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:16,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:18,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:18,939 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:18,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:18,944 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:19,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:20,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:21,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:22,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:22,693 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:22,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:22,698 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:23,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:24,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:24,994 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:05:26,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:28,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:28,426 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:28,426 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:28,432 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:28,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:29,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:29,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:30,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:32,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:35,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:35,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:35,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:35,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:35,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:36,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:38,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:39,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:39,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:39,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:39,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:39,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:40,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:43,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:44,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:44,145 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:44,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:44,150 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:44,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:44,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:46,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:48,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:48,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:48,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:48,949 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:48,954 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:49,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:50,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:53,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:54,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:05:55,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:55,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:55,659 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:55,664 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:56,642 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:56,642 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:59,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:59,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:59,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:59,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:00,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:00,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:00,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:00,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:00,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:02,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:05,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:09,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:09,601 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:09,601 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:09,607 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:09,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:10,608 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:10,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:14,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:14,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:14,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:15,481 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:15,482 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:15,482 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:15,488 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:15,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:16,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:16,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:21,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:21,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:21,226 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:21,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:21,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:22,231 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:22,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:25,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:06:25,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:25,037 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:25,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:25,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:25,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:26,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:28,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:29,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:29,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:29,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:29,278 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:29,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:29,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:29,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:30,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:33,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:33,295 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:33,295 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:33,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:33,300 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:33,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:34,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:37,968 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:37,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:37,969 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:37,974 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:38,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:38,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:40,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:41,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:41,939 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:41,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:41,944 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:42,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:44,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:44,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:44,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:44,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:45,523 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:45,524 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:45,524 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:45,529 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:45,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:46,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:49,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:49,128 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:49,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:49,134 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:49,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:50,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:50,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:55,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:06:56,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:57,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:57,170 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:57,170 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:57,176 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:57,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:58,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:59,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:59,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:59,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:01,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:01,132 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:01,132 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:01,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:01,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:01,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:02,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:05,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:05,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:05,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:05,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:05,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:06,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:06,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:08,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:08,787 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:08,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:08,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:09,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:10,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:11,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:12,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:12,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:12,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:12,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:12,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:14,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:14,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:14,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:14,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:17,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:18,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:18,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:18,145 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:18,150 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:18,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:20,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:21,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:21,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:21,963 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:21,967 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:22,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:22,968 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:24,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:25,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:07:28,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:29,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:29,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:30,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:30,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:30,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:30,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:30,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:32,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:33,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:34,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:34,165 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:34,166 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:34,171 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:34,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:36,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:37,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:37,748 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:37,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:37,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:38,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:38,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:38,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:41,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:41,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:41,516 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:41,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:41,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:42,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:44,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:44,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:44,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:49,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:49,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:49,568 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:49,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:49,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:49,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:50,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:54,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:55,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:07:58,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:58,106 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:58,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:58,112 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:58,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:59,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:59,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:59,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:00,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:00,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:01,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:01,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:01,942 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:01,947 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:02,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:04,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:05,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:06,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:06,436 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:06,436 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:06,442 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:06,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:08,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:10,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:10,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:10,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:10,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:10,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:11,156 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:12,779 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:13,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:13,990 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:13,990 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:13,995 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:14,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:14,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:14,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:14,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:16,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:16,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:20,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:20,411 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:20,411 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:20,416 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:20,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:22,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:22,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:24,464 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:24,465 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:24,465 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:24,471 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:24,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:25,011 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:08:26,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:28,012 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:28,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:28,382 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:28,382 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:28,387 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:28,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:29,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:29,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:30,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:33,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:33,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:33,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:33,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:33,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:33,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:34,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:37,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:37,849 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:37,849 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:37,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:38,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:38,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:38,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:41,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:41,750 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:41,750 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:41,755 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:41,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:42,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:44,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:44,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:44,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:49,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:50,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:50,090 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:50,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:50,095 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:50,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:52,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:53,894 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:53,895 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:53,895 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:53,900 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:54,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:54,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:55,016 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:08:56,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:59,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:59,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:59,204 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:59,209 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:59,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:59,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:59,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:59,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:00,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:00,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:03,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:03,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:03,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:03,391 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:03,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:04,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:06,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:07,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:07,440 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:07,440 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:07,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:07,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:08,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:11,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:12,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:12,199 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:12,199 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:12,204 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:12,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:14,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:14,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:14,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:14,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:16,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:17,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:17,090 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:17,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:17,095 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:17,842 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:18,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:22,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:25,016 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:09:25,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:25,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:25,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:25,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:25,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:26,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:27,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:29,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:29,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:31,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:31,290 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:31,290 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:31,295 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:31,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:32,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:33,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:35,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:35,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:35,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:35,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:35,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:36,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:39,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:39,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:39,988 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:39,988 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:39,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:40,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:42,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:43,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:43,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:43,991 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:43,996 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:44,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:44,867 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:46,870 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:47,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:47,569 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:47,569 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:47,574 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:47,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:48,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:49,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:53,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:53,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:53,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:53,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:53,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:54,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:55,019 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:09:55,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:56,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:56,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:56,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:56,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:56,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:58,883 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:59,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:59,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:59,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:00,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:02,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:02,163 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:02,163 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:02,168 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:02,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:04,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:06,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:07,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:07,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:07,854 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:07,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:07,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:08,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:11,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:14,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:14,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:14,003 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:14,011 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:14,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:14,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:14,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:14,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:16,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:17,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:18,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:18,189 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:18,189 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:18,194 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:18,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:20,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:21,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:21,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:21,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:21,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:21,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:22,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:22,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:25,023 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:10:25,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:25,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:25,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:25,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:26,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:26,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:27,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:29,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:29,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:30,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:30,838 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:30,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:30,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:30,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:32,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:33,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:35,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:35,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:35,626 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:35,631 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:35,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:36,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:39,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:40,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:40,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:40,204 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:40,209 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:40,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:42,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:44,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:44,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:44,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:45,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:49,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:49,343 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:49,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:49,348 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:49,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:50,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:51,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:55,026 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:10:57,027 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:57,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:57,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:57,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:57,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:57,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:58,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:59,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:59,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:02,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:03,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:03,171 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:03,171 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:03,177 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:03,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:04,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:07,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:07,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:07,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:07,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:07,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:08,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:08,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:11,530 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:11,530 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:11,531 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:11,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:11,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:12,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:13,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:14,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:14,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:14,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:15,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:15,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:15,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:15,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:15,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:16,959 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:19,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:19,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:19,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:19,556 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:19,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:19,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:20,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:24,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:25,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:11:27,997 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:27,998 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:27,998 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:28,003 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:28,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:28,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:29,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:29,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:29,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:31,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:31,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:31,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:31,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:31,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:32,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:34,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:39,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:40,538 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:40,539 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:40,539 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:40,544 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:40,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:42,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:44,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:44,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:44,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:44,537 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:44,537 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:44,542 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:44,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:45,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:46,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:50,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:52,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:52,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:52,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:52,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:52,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:54,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:55,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:11:56,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:56,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:56,587 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:56,588 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:56,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:56,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:59,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:59,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:59,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:00,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:00,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:00,378 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:00,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:01,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:01,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:03,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:04,679 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:04,680 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:04,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:04,685 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:05,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:06,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:07,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:08,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:08,973 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:08,974 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:08,979 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:09,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:11,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:11,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:12,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:12,990 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:12,990 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:12,995 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:13,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:14,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:14,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:14,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:15,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:17,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:21,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:21,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:21,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:21,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:22,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:23,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:23,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:25,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:12:26,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:26,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:26,879 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:26,884 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:27,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:28,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:29,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:29,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:29,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:30,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:30,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:30,516 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:30,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:31,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:33,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:34,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:35,012 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:35,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:35,013 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:35,018 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:35,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:37,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:38,760 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:38,760 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:38,761 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:38,765 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:39,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:39,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:41,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:42,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:42,503 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:42,504 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:42,509 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:43,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:44,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:44,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:45,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:45,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:46,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:46,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:46,436 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:46,441 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:47,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:49,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:50,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:50,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:50,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:50,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:50,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:51,027 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:51,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:55,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:12:56,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:56,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:56,946 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:56,946 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:56,951 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:57,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:59,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:59,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:59,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:01,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:01,356 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:01,356 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:01,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:01,361 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:02,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:03,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:05,488 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:05,489 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:05,489 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:05,494 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:06,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:06,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:07,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:09,244 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:09,245 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:09,245 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:09,250 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:10,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:11,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:12,250 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:12,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:12,975 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:12,975 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:12,981 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:13,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:14,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:14,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:14,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:15,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:16,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:16,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:16,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:16,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:17,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:17,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:19,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:21,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:21,690 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:21,691 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:21,695 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:22,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:23,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:23,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:25,041 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:13:26,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:26,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:26,481 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:26,486 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:27,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:29,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:29,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:29,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:29,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:29,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:31,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:31,058 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:31,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:31,063 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:31,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:33,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:35,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:35,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:35,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:35,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:35,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:36,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:37,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:40,027 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:40,028 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:40,028 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:40,033 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:40,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:41,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:41,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:43,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:43,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:43,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:43,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:44,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:44,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:44,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:45,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:46,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:51,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:52,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:52,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:52,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:52,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:53,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:55,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:13:55,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:57,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:59,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:59,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:59,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:59,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:59,917 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:59,917 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:59,922 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:00,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:01,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:02,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:04,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:04,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:04,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:04,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:05,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:07,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:08,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:08,071 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:08,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:08,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:08,076 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:08,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:09,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:13,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:13,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:13,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:13,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:13,321 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:14,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:14,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:14,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:14,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:15,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:17,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:17,356 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:17,356 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:17,361 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:18,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:18,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:19,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:21,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:21,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:21,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:21,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:22,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:23,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:23,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:25,048 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:14:25,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:25,218 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:25,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:25,224 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:26,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:27,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:29,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:29,037 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:29,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:29,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:29,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:29,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:29,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:29,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:29,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:31,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:34,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:34,403 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:34,403 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:34,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:34,408 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:35,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:37,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:38,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:38,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:38,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:38,228 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:39,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:39,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:40,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:42,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:42,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:42,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:42,314 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:43,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:44,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:44,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:45,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:45,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:47,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:47,667 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:47,668 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:47,673 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:48,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:49,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:50,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:51,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:51,437 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:51,438 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:51,442 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:52,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:53,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:55,051 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:14:56,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:57,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:57,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:57,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:57,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:58,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:59,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:59,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:59,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:59,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:01,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:03,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:03,631 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:03,631 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:03,636 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:04,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:05,189 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:06,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:11,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:12,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:12,127 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:12,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:12,132 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:12,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:13,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:14,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:14,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:14,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:17,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:20,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:20,569 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:20,569 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:20,574 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:21,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:23,201 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:23,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:25,054 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:15:26,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:26,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:26,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:26,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:26,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:27,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:29,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:29,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:29,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:29,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:30,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:30,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:30,214 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:30,219 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:31,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:31,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:34,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:35,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:35,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:35,628 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:35,633 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:36,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:37,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:39,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:40,648 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:40,648 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:40,648 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:40,653 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:41,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:43,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:44,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:44,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:44,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:45,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:45,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:45,680 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:45,680 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:45,685 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:46,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:47,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:50,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:50,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:50,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:50,117 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:50,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:51,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:51,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:55,060 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:15:57,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:57,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:57,240 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:57,240 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:57,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:58,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:59,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:59,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:01,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:01,329 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:01,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:01,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:02,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:02,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:03,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:06,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:06,570 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:06,570 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:06,575 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:07,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:07,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:09,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:09,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:09,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:09,949 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:09,954 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:10,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:11,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:12,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:13,960 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:13,961 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:13,961 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:13,966 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:14,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:14,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:14,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:14,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:15,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:18,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:20,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:20,613 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:20,613 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:20,618 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:21,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:23,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:23,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:24,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:24,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:24,626 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:24,631 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:25,059 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:16:25,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:27,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:29,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:29,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:29,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:34,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:34,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:34,370 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:34,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:34,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:35,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:37,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:38,408 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:38,409 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:38,409 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:38,414 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:39,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:39,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:41,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:43,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:43,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:43,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:43,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:44,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:44,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:44,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:44,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:44,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:45,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:47,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:47,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:47,536 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:47,541 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:48,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:49,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:50,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:51,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:51,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:51,926 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:51,931 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:52,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:53,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:55,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:16:56,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:58,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:58,838 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:58,838 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:58,843 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:59,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:59,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:01,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:01,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:04,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:04,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:04,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:04,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:05,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:05,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:07,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:10,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:10,520 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:10,520 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:10,525 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:11,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:12,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:13,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:14,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:14,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:14,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:14,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:14,849 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:14,849 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:14,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:15,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:17,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:17,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:18,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:18,710 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:18,710 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:18,715 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:19,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:21,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:22,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:22,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:22,627 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:22,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:23,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:23,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:25,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:17:25,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:29,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:29,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:29,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:29,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:31,810 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:31,811 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:31,811 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:31,816 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:32,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:33,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:34,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:35,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:35,571 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:35,572 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:35,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:36,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:37,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:39,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:39,743 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:39,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:39,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:40,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:40,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:41,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:43,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:43,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:43,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:43,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:44,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:44,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:44,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:44,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:45,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:46,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:49,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:49,186 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:49,186 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:49,191 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:49,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:51,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:52,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:52,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:52,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:52,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:52,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:53,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:55,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:17:55,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:57,200 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:57,201 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:57,201 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:57,206 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:57,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:58,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:59,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:59,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:01,281 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:01,281 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:01,281 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:01,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:01,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:03,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:03,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:06,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:06,983 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:06,983 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:06,988 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:07,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:08,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:09,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:12,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:12,456 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:12,456 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:12,461 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:13,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:13,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:14,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:14,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:14,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:14,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:16,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:16,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:16,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:16,911 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:17,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:19,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:19,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:22,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:22,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:22,040 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:22,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:22,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:23,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:25,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:25,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:18:26,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:26,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:26,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:26,041 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:26,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:27,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:29,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:29,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:29,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:29,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:29,869 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:29,870 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:29,875 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:30,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:30,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:31,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:34,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:34,563 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:34,563 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:34,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:35,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:36,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:37,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:38,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:38,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:38,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:38,396 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:38,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:39,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:42,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:44,034 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:44,035 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:44,035 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:44,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:44,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:44,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:44,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:44,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:45,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:47,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:49,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:49,155 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:49,155 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:49,160 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:49,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:51,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:53,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:53,177 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:53,177 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:53,178 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:53,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:53,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:55,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:18:55,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:57,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:57,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:57,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:57,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:57,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:59,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:59,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:59,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:59,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:59,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:00,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:00,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:00,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:00,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:01,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:03,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:04,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:04,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:04,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:04,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:04,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:05,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:07,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:08,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:08,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:08,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:08,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:09,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:09,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:10,520 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:14,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:14,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:14,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:15,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:15,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:15,030 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:15,034 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:15,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:16,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:17,442 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:19,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:19,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:19,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:19,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:20,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:21,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:21,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:25,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:19:27,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:28,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:28,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:28,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:28,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:28,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:29,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:29,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:29,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:29,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:32,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:32,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:32,246 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:32,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:32,251 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:32,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:33,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:36,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:36,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:36,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:36,212 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:36,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:37,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:38,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:39,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:39,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:39,796 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:39,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:40,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:41,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:43,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:44,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:44,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:44,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:45,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:45,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:45,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:45,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:45,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:47,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:49,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:50,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:50,391 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:50,391 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:50,396 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:50,477 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:51,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:54,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:54,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:54,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:54,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:54,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:55,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:19:55,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:57,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:59,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:59,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:00,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:00,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:00,239 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:00,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:00,244 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:00,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:01,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:04,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:04,620 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:04,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:04,626 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:05,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:05,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:05,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:08,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:08,564 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:08,564 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:08,569 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:09,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:09,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:11,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:14,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:14,278 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:14,278 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:14,283 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:14,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:14,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:14,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:14,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:15,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:16,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:18,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:18,299 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:18,299 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:18,304 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:18,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:19,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:21,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:21,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:21,854 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:21,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:21,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:22,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:23,511 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:25,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:20:26,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:26,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:26,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:26,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:26,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:27,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:27,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:29,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:29,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:29,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:32,042 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:32,043 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:32,043 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:32,048 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:32,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:33,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:33,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:37,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:37,513 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:37,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:37,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:37,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:38,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:39,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:41,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:41,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:41,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:41,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:41,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:43,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:43,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:44,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:44,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:44,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:45,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:45,459 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:45,459 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:45,464 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:45,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:47,536 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:49,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:49,023 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:49,024 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:49,025 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:49,029 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:49,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:51,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:54,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:55,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:20:57,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:57,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:57,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:57,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:57,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:59,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:59,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:59,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:59,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:01,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:01,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:01,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:01,269 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:01,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:03,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:05,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:05,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:05,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:05,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:05,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:05,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:07,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:09,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:09,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:09,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:09,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:09,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:11,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:11,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:12,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:12,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:12,992 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:12,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:13,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:14,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:14,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:14,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:15,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:16,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:21,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:21,411 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:21,411 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:21,417 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:21,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:22,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:23,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:25,092 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:21:25,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:25,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:25,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:25,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:26,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:27,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:27,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:29,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:29,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:29,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:29,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:29,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:29,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:29,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:30,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:31,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:33,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:33,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:33,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:33,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:33,364 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:33,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:35,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:37,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:37,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:37,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:37,802 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:38,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:38,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:39,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:43,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:43,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:43,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:43,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:43,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:44,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:44,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:44,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:44,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:45,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:47,200 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:47,201 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:47,201 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:47,208 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:47,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:49,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:50,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:51,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:51,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:51,004 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:51,008 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:51,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:53,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:55,092 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:21:55,824 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:55,825 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:55,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:55,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:55,830 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:56,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:57,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:59,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:01,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:04,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:04,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:04,279 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:04,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:04,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:05,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:07,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:08,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:08,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:08,875 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:08,880 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:09,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:11,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:12,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:12,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:12,712 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:12,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:12,717 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:13,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:13,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:14,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:14,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:14,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:17,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:17,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:17,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:17,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:17,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:18,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:19,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:23,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:25,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:22:26,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:26,168 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:26,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:26,173 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:26,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:27,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:29,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:29,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:29,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:29,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:31,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:31,396 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:31,396 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:31,401 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:31,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:33,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:34,444 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:37,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:37,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:37,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:37,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:37,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:39,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:40,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:40,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:40,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:40,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:40,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:41,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:43,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:44,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:44,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:44,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:44,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:44,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:44,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:44,596 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:44,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:45,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:45,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:48,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:48,584 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:48,584 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:48,589 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:48,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:49,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:51,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:52,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:52,571 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:52,571 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:52,576 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:52,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:53,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:55,099 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:22:57,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:58,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:58,050 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:58,050 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:58,055 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:58,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:59,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:59,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:59,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:02,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:03,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:03,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:03,742 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:03,747 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:04,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:05,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:07,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:12,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:12,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:12,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:12,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:12,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:13,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:13,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:14,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:14,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:14,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:15,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:15,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:15,749 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:15,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:16,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:17,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:18,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:21,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:21,616 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:21,616 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:21,621 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:21,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:23,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:24,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:25,102 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:23:25,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:25,464 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:25,464 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:25,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:25,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:27,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:29,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:29,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:29,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:30,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:33,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:33,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:33,927 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:33,932 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:34,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:35,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:35,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:40,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:40,175 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:40,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:40,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:40,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:41,180 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:41,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:43,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:43,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:43,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:43,452 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:43,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:44,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:44,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:44,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:45,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:46,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:48,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:48,862 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:48,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:48,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:49,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:51,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:51,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:55,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:23:57,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:57,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:57,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:57,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:57,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:57,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:59,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:59,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:59,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:59,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:01,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:04,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:04,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:04,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:04,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:04,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:05,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:05,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:05,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:05,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:05,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:05,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:07,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:08,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:08,587 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:08,587 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:08,592 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:08,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:09,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:09,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:13,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:13,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:13,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:13,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:13,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:14,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:14,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:14,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:15,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:15,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:18,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:18,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:18,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:18,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:19,750 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:20,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:21,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:22,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:22,480 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:22,480 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:22,486 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:22,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:23,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:25,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:24:25,282 WARNING FileStreamThread:1827008 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/lilei_stones/hf-vlms/zo3nkbji/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/lilei_stones/hf-vlms/zo3nkbji/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"Finetune/Step": 2113, "Finetune/Loss": 0.8899515867233276, "Finetune/Loss (Raw)": 0.9392950534820557, "Finetune/Learning Rate": 1.3438732176917437e-05, "Finetune/Step Time": 4.916087483987212, "_timestamp": 1710264195.7488248, "_runtime": 10462.06027674675, "_step": 2113}']}, 'wandb-history.jsonl': {'offset': 2108, 'content': ['{"Finetune/Step": 2109, "Finetune/Loss": 0.8953642249107361, "Finetune/Loss (Raw)": 0.9452600479125977, "Finetune/Learning Rate": 1.3462120295134476e-05, "Finetune/Step Time": 4.860195890069008, "_timestamp": 1710264172.5707068, "_runtime": 10438.882158756256, "_step": 2109}', '{"Finetune/Step": 2110, "Finetune/Loss": 0.8953254222869873, "Finetune/Loss (Raw)": 0.962855875492096, "Finetune/Learning Rate": 1.3456275275280661e-05, "Finetune/Step Time": 4.868157049641013, "_timestamp": 1710264178.0491347, "_runtime": 10444.360586643219, "_step": 2110}', '{"Finetune/Step": 2111, "Finetune/Loss": 0.8954590559005737, "Finetune/Loss (Raw)": 0.9165306687355042, "Finetune/Learning Rate": 1.3450428914113892e-05, "Finetune/Step Time": 4.882664466276765, "_timestamp": 1710264183.7410262, "_runtime": 10450.052478075027, "_step": 2111}', '{"Finetune/Step": 2112, "Finetune/Loss": 0.890485405921936, "Finetune/Loss (Raw)": 0.4575220048427582, "Finetune/Learning Rate": 1.3444581213903026e-05, "Finetune/Step Time": 4.918228147551417, "_timestamp": 1710264192.0731206, "_runtime": 10458.384572505951, "_step": 2112}', '{"Finetune/Step": 2113, "Finetune/Loss": 0.8899515867233276, "Finetune/Loss (Raw)": 0.9392950534820557, "Finetune/Learning Rate": 1.3438732176917437e-05, "Finetune/Step Time": 4.916087483987212, "_timestamp": 1710264195.7488248, "_runtime": 10462.06027674675, "_step": 2113}']}, 'wandb-events.jsonl': {'offset': 347, 'content': ['{"system.proc.memory.availableMB": 1652372.86, "system.memory": 19.9, "system.proc.memory.rssMB": 8296.52, "system.proc.memory.percent": 0.4, "system.network.sent": 6265215864.67, "system.network.recv": 29959944514.6, "system.cpu": 0.39, "system.cpu.0.cpu_percent": 0.07, "system.cpu.1.cpu_percent": 0.0, "system.cpu.2.cpu_percent": 0.03, "system.cpu.3.cpu_percent": 0.17, "system.cpu.4.cpu_percent": 0.1, "system.cpu.5.cpu_percent": 1.2, "system.cpu.6.cpu_percent": 67.55, "system.cpu.7.cpu_percent": 0.0, "system.cpu.8.cpu_percent": 0.03, "system.cpu.9.cpu_percent": 32.02, "system.cpu.10.cpu_percent": 0.07, "system.cpu.11.cpu_percent": 0.63, "system.cpu.12.cpu_percent": 1.2, "system.cpu.13.cpu_percent": 0.2, "system.cpu.14.cpu_percent": 0.03, "system.cpu.15.cpu_percent": 2.66, "system.cpu.16.cpu_percent": 0.63, "system.cpu.17.cpu_percent": 1.07, "system.cpu.18.cpu_percent": 0.27, "system.cpu.19.cpu_percent": 0.07, "system.cpu.20.cpu_percent": 0.07, "system.cpu.21.cpu_percent": 0.53, "system.cpu.22.cpu_percent": 0.6, "system.cpu.23.cpu_percent": 0.07, "system.cpu.24.cpu_percent": 31.73, "system.cpu.25.cpu_percent": 1.23, "system.cpu.26.cpu_percent": 0.07, "system.cpu.27.cpu_percent": 0.1, "system.cpu.28.cpu_percent": 0.07, "system.cpu.29.cpu_percent": 0.2, "system.cpu.30.cpu_percent": 0.07, "system.cpu.31.cpu_percent": 68.27, "system.cpu.32.cpu_percent": 0.0, "system.cpu.33.cpu_percent": 0.0, "system.cpu.34.cpu_percent": 32.35, "system.cpu.35.cpu_percent": 0.0, "system.cpu.36.cpu_percent": 0.03, "system.cpu.37.cpu_percent": 1.2, "system.cpu.38.cpu_percent": 0.2, "system.cpu.39.cpu_percent": 1.53, "system.cpu.40.cpu_percent": 0.13, "system.cpu.41.cpu_percent": 32.23, "system.cpu.42.cpu_percent": 0.0, "system.cpu.43.cpu_percent": 0.0, "system.cpu.44.cpu_percent": 0.03, "system.cpu.45.cpu_percent": 0.07, "system.cpu.46.cpu_percent": 0.5, "system.cpu.47.cpu_percent": 69.47, "system.cpu.48.cpu_percent": 12.12, "system.cpu.49.cpu_percent": 0.1, "system.cpu.50.cpu_percent": 0.0, "system.cpu.51.cpu_percent": 0.57, "system.cpu.52.cpu_percent": 0.83, "system.cpu.53.cpu_percent": 0.1, "system.cpu.54.cpu_percent": 0.07, "system.cpu.55.cpu_percent": 0.2, "system.cpu.56.cpu_percent": 20.01, "system.cpu.57.cpu_percent": 0.0, "system.cpu.58.cpu_percent": 0.0, "system.cpu.59.cpu_percent": 0.63, "system.cpu.60.cpu_percent": 0.6, "system.cpu.61.cpu_percent": 0.0, "system.cpu.62.cpu_percent": 0.13, "system.cpu.63.cpu_percent": 0.17, "system.cpu.64.cpu_percent": 0.0, "system.cpu.65.cpu_percent": 0.07, "system.cpu.66.cpu_percent": 0.0, "system.cpu.67.cpu_percent": 0.0, "system.cpu.68.cpu_percent": 0.0, "system.cpu.69.cpu_percent": 0.0, "system.cpu.70.cpu_percent": 0.17, "system.cpu.71.cpu_percent": 0.0, "system.cpu.72.cpu_percent": 0.0, "system.cpu.73.cpu_percent": 0.0, "system.cpu.74.cpu_percent": 0.0, "system.cpu.75.cpu_percent": 0.0, "system.cpu.76.cpu_percent": 0.0, "system.cpu.77.cpu_percent": 0.0, "system.cpu.78.cpu_percent": 0.0, "system.cpu.79.cpu_percent": 0.0, "system.cpu.80.cpu_percent": 0.03, "system.cpu.81.cpu_percent": 0.0, "system.cpu.82.cpu_percent": 0.03, "system.cpu.83.cpu_percent": 0.03, "system.cpu.84.cpu_percent": 0.0, "system.cpu.85.cpu_percent": 0.9, "system.cpu.86.cpu_percent": 0.03, "system.cpu.87.cpu_percent": 0.0, "system.cpu.88.cpu_percent": 0.0, "system.cpu.89.cpu_percent": 0.03, "system.cpu.90.cpu_percent": 0.0, "system.cpu.91.cpu_percent": 0.0, "system.cpu.92.cpu_percent": 0.07, "system.cpu.93.cpu_percent": 0.03, "system.cpu.94.cpu_percent": 0.0, "system.cpu.95.cpu_percent": 0.07, "system.cpu.96.cpu_percent": 0.0, "system.cpu.97.cpu_percent": 4.5, "system.cpu.98.cpu_percent": 0.0, "system.cpu.99.cpu_percent": 0.0, "system.cpu.100.cpu_percent": 0.03, "system.cpu.101.cpu_percent": 0.0, "system.cpu.102.cpu_percent": 0.0, "system.cpu.103.cpu_percent": 0.0, "system.cpu.104.cpu_percent": 3.23, "system.cpu.105.cpu_percent": 1.66, "system.cpu.106.cpu_percent": 1.46, "system.cpu.107.cpu_percent": 0.0, "system.cpu.108.cpu_percent": 1.6, "system.cpu.109.cpu_percent": 0.0, "system.cpu.110.cpu_percent": 0.0, "system.cpu.111.cpu_percent": 0.0, "system.cpu.112.cpu_percent": 0.0, "system.cpu.113.cpu_percent": 0.0, "system.cpu.114.cpu_percent": 0.0, "system.cpu.115.cpu_percent": 0.0, "system.cpu.116.cpu_percent": 0.0, "system.cpu.117.cpu_percent": 0.0, "system.cpu.118.cpu_percent": 0.0, "system.cpu.119.cpu_percent": 0.03, "system.cpu.120.cpu_percent": 0.0, "system.cpu.121.cpu_percent": 0.03, "system.cpu.122.cpu_percent": 0.0, "system.cpu.123.cpu_percent": 0.03, "system.cpu.124.cpu_percent": 0.0, "system.cpu.125.cpu_percent": 0.6, "system.cpu.126.cpu_percent": 0.0, "system.cpu.127.cpu_percent": 0.03, "system.cpu.128.cpu_percent": 0.03, "system.cpu.129.cpu_percent": 0.03, "system.cpu.130.cpu_percent": 0.2, "system.cpu.131.cpu_percent": 32.37, "system.cpu.132.cpu_percent": 0.0, "system.cpu.133.cpu_percent": 1.33, "system.cpu.134.cpu_percent": 0.0, "system.cpu.135.cpu_percent": 0.0, "system.cpu.136.cpu_percent": 0.03, "system.cpu.137.cpu_percent": 0.13, "system.cpu.138.cpu_percent": 67.95, "system.cpu.139.cpu_percent": 0.07, "system.cpu.140.cpu_percent": 0.0, "system.cpu.141.cpu_percent": 0.03, "system.cpu.142.cpu_percent": 0.67, "system.cpu.143.cpu_percent": 7.65, "system.cpu.144.cpu_percent": 0.0, "system.cpu.145.cpu_percent": 0.0, "system.cpu.146.cpu_percent": 0.0, "system.cpu.147.cpu_percent": 0.03, "system.cpu.148.cpu_percent": 0.07, "system.cpu.149.cpu_percent": 0.07, "system.cpu.150.cpu_percent": 0.0, "system.cpu.151.cpu_percent": 0.0, "system.cpu.152.cpu_percent": 0.1, "system.cpu.153.cpu_percent": 0.03, "system.cpu.154.cpu_percent": 0.07, "system.cpu.155.cpu_percent": 0.0, "system.cpu.156.cpu_percent": 0.0, "system.cpu.157.cpu_percent": 0.1, "system.cpu.158.cpu_percent": 0.0, "system.cpu.159.cpu_percent": 0.0, "system.cpu.160.cpu_percent": 32.39, "system.cpu.161.cpu_percent": 67.7, "system.cpu.162.cpu_percent": 0.0, "system.cpu.163.cpu_percent": 0.2, "system.cpu.164.cpu_percent": 0.7, "system.cpu.165.cpu_percent": 0.77, "system.cpu.166.cpu_percent": 0.03, "system.cpu.167.cpu_percent": 67.59, "system.cpu.168.cpu_percent": 0.6, "system.cpu.169.cpu_percent": 0.03, "system.cpu.170.cpu_percent": 31.04, "system.cpu.171.cpu_percent": 0.17, "system.cpu.172.cpu_percent": 67.91, "system.cpu.173.cpu_percent": 1.23, "system.cpu.174.cpu_percent": 0.17, "system.cpu.175.cpu_percent": 0.1, "system.cpu.176.cpu_percent": 0.03, "system.cpu.177.cpu_percent": 0.03, "system.cpu.178.cpu_percent": 0.0, "system.cpu.179.cpu_percent": 0.1, "system.cpu.180.cpu_percent": 0.03, "system.cpu.181.cpu_percent": 0.13, "system.cpu.182.cpu_percent": 1.27, "system.cpu.183.cpu_percent": 0.07, "system.cpu.184.cpu_percent": 0.03, "system.cpu.185.cpu_percent": 0.1, "system.cpu.186.cpu_percent": 0.17, "system.cpu.187.cpu_percent": 67.97, "system.cpu.188.cpu_percent": 0.13, "system.cpu.189.cpu_percent": 0.1, "system.cpu.190.cpu_percent": 0.13, "system.cpu.191.cpu_percent": 0.0, "system.cpu.192.cpu_percent": 0.0, "system.cpu.193.cpu_percent": 0.03, "system.cpu.194.cpu_percent": 0.0, "system.cpu.195.cpu_percent": 0.03, "system.cpu.196.cpu_percent": 0.07, "system.cpu.197.cpu_percent": 0.1, "system.cpu.198.cpu_percent": 0.0, "system.cpu.199.cpu_percent": 0.03, "system.cpu.200.cpu_percent": 0.0, "system.cpu.201.cpu_percent": 0.0, "system.cpu.202.cpu_percent": 0.0, "system.cpu.203.cpu_percent": 0.03, "system.cpu.204.cpu_percent": 0.0, "system.cpu.205.cpu_percent": 0.0, "system.cpu.206.cpu_percent": 0.0, "system.cpu.207.cpu_percent": 0.0, "system.cpu.208.cpu_percent": 0.0, "system.cpu.209.cpu_percent": 0.0, "system.cpu.210.cpu_percent": 0.0, "system.cpu.211.cpu_percent": 0.0, "system.cpu.212.cpu_percent": 0.03, "system.cpu.213.cpu_percent": 0.07, "system.cpu.214.cpu_percent": 0.03, "system.cpu.215.cpu_percent": 0.0, "system.cpu.216.cpu_percent": 0.1, "system.cpu.217.cpu_percent": 0.0, "system.cpu.218.cpu_percent": 0.0, "system.cpu.219.cpu_percent": 0.0, "system.cpu.220.cpu_percent": 0.0, "system.cpu.221.cpu_percent": 0.03, "system.cpu.222.cpu_percent": 0.03, "system.cpu.223.cpu_percent": 0.0, "system.cpu.224.cpu_percent": 0.0, "system.cpu.225.cpu_percent": 0.0, "system.cpu.226.cpu_percent": 0.0, "system.cpu.227.cpu_percent": 0.0, "system.cpu.228.cpu_percent": 0.0, "system.cpu.229.cpu_percent": 0.0, "system.cpu.230.cpu_percent": 4.66, "system.cpu.231.cpu_percent": 0.0, "system.cpu.232.cpu_percent": 0.0, "system.cpu.233.cpu_percent": 0.0, "system.cpu.234.cpu_percent": 0.0, "system.cpu.235.cpu_percent": 0.0, "system.cpu.236.cpu_percent": 0.5, "system.cpu.237.cpu_percent": 0.0, "system.cpu.238.cpu_percent": 0.0, "system.cpu.239.cpu_percent": 1.46, "system.cpu.240.cpu_percent": 0.0, "system.cpu.241.cpu_percent": 0.03, "system.cpu.242.cpu_percent": 0.07, "system.cpu.243.cpu_percent": 0.03, "system.cpu.244.cpu_percent": 0.0, "system.cpu.245.cpu_percent": 0.0, "system.cpu.246.cpu_percent": 0.0, "system.cpu.247.cpu_percent": 0.03, "system.cpu.248.cpu_percent": 0.0, "system.cpu.249.cpu_percent": 0.03, "system.cpu.250.cpu_percent": 0.0, "system.cpu.251.cpu_percent": 0.0, "system.cpu.252.cpu_percent": 0.03, "system.cpu.253.cpu_percent": 0.0, "system.cpu.254.cpu_percent": 0.03, "system.proc.cpu.threads": 45, "system.disk.\\\\.usagePercent": 22.9, "system.disk.\\\\.usageGB": 55.32, "system.disk.in": 64.2, "system.disk.out": 567.52, "system.gpu.0.memoryAllocated": 99.19, "system.gpu.process.0.memoryAllocated": 99.19, "system.gpu.1.memoryAllocated": 81.67, "system.gpu.2.memoryAllocated": 80.95, "system.gpu.3.memoryAllocated": 78.9, "system.gpu.4.memoryAllocated": 79.57, "system.gpu.5.memoryAllocated": 87.99, "system.gpu.6.memoryAllocated": 82.35, "system.gpu.7.memoryAllocated": 91.58, "system.gpu.0.memoryAllocatedBytes": 85206630400.0, "system.gpu.process.0.memoryAllocatedBytes": 85206630400.0, "system.gpu.1.memoryAllocatedBytes": 70155370496.0, "system.gpu.2.memoryAllocatedBytes": 69536710656.0, "system.gpu.3.memoryAllocatedBytes": 67777200128.0, "system.gpu.4.memoryAllocatedBytes": 68353916928.0, "system.gpu.5.memoryAllocatedBytes": 75586994176.0, "system.gpu.6.memoryAllocatedBytes": 70738378752.0, "system.gpu.7.memoryAllocatedBytes": 78669807616.0, "system.gpu.0.memory": 38.73, "system.gpu.process.0.memory": 38.73, "system.gpu.1.memory": 34.93, "system.gpu.2.memory": 32.67, "system.gpu.3.memory": 30.73, "system.gpu.4.memory": 30.73, "system.gpu.5.memory": 32.47, "system.gpu.6.memory": 29.8, "system.gpu.7.memory": 31.07, "system.gpu.0.gpu": 99.93, "system.gpu.process.0.gpu": 99.93, "system.gpu.1.gpu": 99.93, "system.gpu.2.gpu": 99.93, "system.gpu.3.gpu": 99.93, "system.gpu.4.gpu": 99.93, "system.gpu.5.gpu": 99.93, "system.gpu.6.gpu": 99.93, "system.gpu.7.gpu": 99.93, "system.gpu.0.temp": 73.6, "system.gpu.process.0.temp": 73.6, "system.gpu.1.temp": 63.8, "system.gpu.2.temp": 66.47, "system.gpu.3.temp": 73.93, "system.gpu.4.temp": 69.73, "system.gpu.5.temp": 64.4, "system.gpu.6.temp": 65.8, "system.gpu.7.temp": 74.0, "system.gpu.0.powerWatts": 369.2, "system.gpu.process.0.powerWatts": 369.2, "system.gpu.1.powerWatts": 368.79, "system.gpu.2.powerWatts": 331.63, "system.gpu.3.powerWatts": 316.25, "system.gpu.4.powerWatts": 334.68, "system.gpu.5.powerWatts": 337.44, "system.gpu.6.powerWatts": 338.5, "system.gpu.7.powerWatts": 322.51, "system.gpu.0.powerPercent": 91.41, "system.gpu.process.0.powerPercent": 91.41, "system.gpu.1.powerPercent": 89.03, "system.gpu.2.powerPercent": 85.94, "system.gpu.3.powerPercent": 73.85, "system.gpu.4.powerPercent": 82.96, "system.gpu.5.powerPercent": 87.93, "system.gpu.6.powerPercent": 84.56, "system.gpu.7.powerPercent": 83.14, "_wandb": true, "_timestamp": 1710264205.099411, "_runtime": 10471.410863}']}, 'output.log': {'offset': 1, 'content': ['ERROR 2024-03-12T17:23:22.980327 =>> [Global Step] 002114 =>> LR :: 0.000013 -- Loss :: 0.8894: 41%|█████████████████▍ | 2114/5198 [2:54:19<4:41:06, 5.47s/it]\n']}}, 'dropped': 0}} +2024-03-12 17:24:26,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:26,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:26,668 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:26,668 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:26,673 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:26,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:27,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:29,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:29,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:29,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:31,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:35,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:35,610 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:35,610 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:35,614 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:35,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:36,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:37,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:41,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:43,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:43,948 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:43,949 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:43,953 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:44,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:44,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:44,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:44,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:45,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:47,472 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:47,472 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:47,473 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:47,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:47,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:47,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:49,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:51,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:51,414 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:51,414 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:51,419 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:51,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:53,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:53,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:55,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:24:55,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:55,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:55,991 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:55,996 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:56,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:57,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:58,997 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:59,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:59,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:59,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:59,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:59,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:59,796 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:59,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:00,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:01,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:04,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:04,894 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:04,894 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:04,894 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:04,900 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:05,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:05,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:08,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:08,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:08,169 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:08,174 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:08,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:09,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:10,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:11,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:11,780 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:11,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:11,785 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:11,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:13,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:14,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:14,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:14,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:15,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:20,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:20,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:20,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:20,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:20,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:21,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:21,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:24,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:24,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:24,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:24,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:24,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:25,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:25:25,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:27,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:29,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:29,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:30,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:30,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:30,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:30,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:30,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:31,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:32,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:35,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:35,494 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:35,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:35,500 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:35,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:37,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:38,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:40,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:40,020 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:40,020 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:40,025 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:40,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:41,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:44,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:44,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:44,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:44,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:44,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:44,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:44,628 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:44,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:44,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:45,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:49,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:51,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:51,071 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:51,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:51,076 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:51,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:53,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:55,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:55,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:25:57,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:57,172 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:57,172 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:57,177 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:57,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:59,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:59,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:59,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:59,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:00,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:02,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:02,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:02,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:02,799 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:02,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:03,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:05,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:08,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:08,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:08,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:08,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:09,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:09,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:10,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:13,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:13,740 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:13,740 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:13,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:13,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:14,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:14,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:14,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:15,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:16,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:17,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:17,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:17,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:17,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:17,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:19,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:21,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:21,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:21,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:21,965 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:21,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:22,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:23,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:25,119 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:26:26,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:26,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:26,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:26,124 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:26,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:27,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:27,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:29,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:29,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:31,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:31,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:31,204 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:31,209 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:31,883 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:32,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:33,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:36,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:36,019 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:36,020 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:36,025 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:36,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:37,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:38,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:39,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:39,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:39,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:39,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:40,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:41,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:43,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:44,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:44,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:44,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:44,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:44,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:44,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:44,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:44,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:45,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:49,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:49,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:49,496 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:49,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:49,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:49,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:51,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:54,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:54,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:54,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:54,919 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:54,924 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:55,121 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:26:55,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:55,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:59,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:59,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:59,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:00,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:00,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:00,824 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:00,828 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:00,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:01,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:04,760 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:04,761 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:04,761 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:04,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:04,766 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:04,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:05,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:09,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:10,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:10,847 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:10,847 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:10,852 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:10,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:11,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:14,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:14,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:14,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:14,483 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:14,543 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:14,543 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:14,547 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:14,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:15,548 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:15,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:18,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:18,092 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:18,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:18,097 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:18,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:19,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:21,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:23,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:23,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:23,682 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:23,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:23,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:25,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:27:25,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:27,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:29,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:29,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:29,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:29,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:29,613 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:29,613 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:29,618 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:29,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:31,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:32,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:33,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:33,182 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:33,182 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:33,187 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:33,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:35,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:38,188 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:40,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:40,584 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:40,584 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:40,589 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:40,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:41,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:43,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:44,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:44,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:44,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:44,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:44,933 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:44,933 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:44,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:44,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:45,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:48,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:48,714 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:48,714 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:48,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:48,719 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:48,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:49,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:53,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:53,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:53,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:53,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:53,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:54,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:55,127 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:27:55,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:56,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:56,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:56,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:56,845 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:56,979 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:57,982 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:59,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:59,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:59,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:00,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:00,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:00,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:00,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:00,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:01,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:04,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:06,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:06,181 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:06,182 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:06,186 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:06,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:07,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:10,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:10,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:10,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:10,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:10,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:10,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:12,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:14,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:14,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:14,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:15,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:16,156 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:16,156 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:16,156 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:16,161 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:17,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:18,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:21,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:21,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:21,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:21,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:21,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:22,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:24,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:25,133 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:28:27,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:27,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:27,815 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:27,815 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:27,820 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:28,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:29,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:29,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:29,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:30,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:31,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:31,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:31,865 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:31,870 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:32,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:32,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:34,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:36,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:36,220 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:36,220 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:36,225 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:37,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:38,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:38,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:40,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:40,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:40,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:40,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:41,043 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:42,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:44,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:44,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:44,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:44,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:45,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:45,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:45,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:45,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:46,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:48,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:49,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:50,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:50,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:50,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:50,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:51,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:52,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:53,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:53,886 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:53,886 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:53,891 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:54,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:54,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:55,133 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:28:56,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:58,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:58,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:58,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:58,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:59,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:59,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:59,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:00,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:00,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:04,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:04,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:04,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:04,915 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:05,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:05,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:06,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:10,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:10,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:10,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:10,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:11,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:11,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:12,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:14,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:14,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:14,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:14,597 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:14,597 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:14,602 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:15,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:16,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:17,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:17,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:17,814 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:17,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:17,819 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:18,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:20,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:22,820 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:25,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:29:26,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:26,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:26,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:26,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:27,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:28,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:28,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:29,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:29,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:33,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:34,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:34,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:34,298 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:34,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:35,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:36,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:39,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:39,194 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:39,194 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:39,194 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:39,199 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:40,116 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:40,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:44,200 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:44,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:44,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:44,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:44,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:44,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:44,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:45,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:46,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:49,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:49,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:49,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:49,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:49,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:50,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:52,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:54,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:54,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:54,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:54,158 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:55,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:55,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:29:55,139 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:56,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:58,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:58,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:58,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:58,468 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:59,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:59,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:59,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:00,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:00,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:02,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:02,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:02,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:02,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:03,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:04,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:06,468 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:06,469 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:06,469 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:06,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:06,474 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:07,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:08,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:11,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:11,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:11,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:11,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:11,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:12,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:14,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:14,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:14,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:15,483 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:15,484 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:15,484 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:15,489 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:16,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:16,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:18,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:20,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:20,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:20,298 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:20,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:21,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:22,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:22,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:24,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:24,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:24,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:24,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:24,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:25,143 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:30:26,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:28,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:29,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:29,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:29,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:29,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:29,629 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:29,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:30,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:32,168 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:33,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:37,816 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:37,817 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:37,817 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:37,822 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:38,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:38,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:40,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:41,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:41,117 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:41,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:41,122 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:41,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:42,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:44,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:44,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:44,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:45,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:45,131 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:45,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:45,136 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:45,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:46,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:49,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:50,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:50,501 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:50,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:50,506 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:51,187 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:52,189 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:54,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:55,143 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:30:59,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:59,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:59,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:59,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:59,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:59,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:59,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:59,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:00,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:02,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:04,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:09,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:10,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:10,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:10,795 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:10,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:11,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:12,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:14,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:14,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:15,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:15,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:15,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:15,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:15,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:16,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:16,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:20,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:20,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:20,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:20,144 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:20,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:21,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:22,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:24,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:24,276 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:24,276 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:24,282 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:25,145 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:31:25,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:26,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:26,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:29,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:29,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:31,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:33,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:33,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:33,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:33,252 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:34,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:34,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:37,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:40,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:40,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:40,916 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:40,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:41,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:42,232 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:42,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:44,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:44,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:47,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:47,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:47,420 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:47,425 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:48,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:48,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:50,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:51,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:51,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:51,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:51,915 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:52,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:53,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:54,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:55,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:31:56,839 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:56,840 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:56,840 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:56,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:57,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:58,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:59,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:59,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:00,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:00,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:00,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:00,337 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:01,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:02,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:04,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:04,270 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:04,270 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:04,275 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:05,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:05,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:06,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:08,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:08,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:08,257 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:08,262 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:09,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:10,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:11,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:13,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:13,502 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:13,503 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:13,507 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:14,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:14,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:14,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:16,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:16,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:21,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:21,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:21,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:21,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:21,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:22,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:24,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:25,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:32:25,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:25,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:25,554 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:25,560 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:26,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:27,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:28,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:29,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:29,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:29,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:31,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:31,478 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:31,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:31,483 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:32,280 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:33,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:34,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:36,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:36,329 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:36,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:36,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:37,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:38,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:39,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:41,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:41,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:41,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:41,057 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:41,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:42,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:44,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:44,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:44,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:44,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:45,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:45,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:45,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:45,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:45,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:46,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:50,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:50,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:50,290 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:50,290 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:50,295 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:50,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:52,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:55,158 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:32:55,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:56,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:56,529 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:56,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:56,534 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:57,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:58,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:59,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:59,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:00,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:01,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:01,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:01,018 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:01,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:01,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:02,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:06,035 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:06,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:06,036 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:06,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:06,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:06,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:08,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:09,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:09,802 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:09,802 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:09,807 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:10,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:11,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:12,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:14,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:14,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:14,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:14,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:14,793 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:14,793 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:14,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:15,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:16,328 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:16,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:18,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:18,718 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:18,719 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:18,723 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:19,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:20,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:22,724 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:22,831 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:22,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:22,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:22,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:23,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:24,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:25,158 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:33:28,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:28,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:28,211 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:28,212 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:28,216 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:28,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:29,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:29,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:29,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:30,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:32,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:32,533 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:32,533 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:32,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:33,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:33,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:34,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:37,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:37,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:37,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:37,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:38,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:38,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:40,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:41,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:41,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:41,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:41,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:42,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:43,819 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:44,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:44,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:44,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:44,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:45,984 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:45,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:45,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:45,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:46,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:48,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:49,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:51,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:51,966 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:51,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:51,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:52,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:54,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:54,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:55,164 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:33:55,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:55,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:55,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:55,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:56,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:58,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:59,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:59,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:59,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:59,901 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:59,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:59,907 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:00,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:00,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:02,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:04,670 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:04,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:04,671 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:04,676 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:05,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:06,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:06,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:08,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:08,325 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:08,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:08,330 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:08,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:10,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:12,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:12,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:12,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:12,516 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:12,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:13,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:14,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:14,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:14,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:14,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:16,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:16,175 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:16,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:16,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:16,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:18,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:18,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:19,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:19,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:19,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:19,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:20,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:22,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:23,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:23,504 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:23,504 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:23,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:23,509 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:24,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:24,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:25,164 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:34:28,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:28,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:28,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:28,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:28,779 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:29,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:29,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:29,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:29,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:30,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:32,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:32,229 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:32,229 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:32,234 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:32,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:34,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:34,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:36,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:36,011 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:36,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:36,016 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:36,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:38,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:40,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:42,735 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:42,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:42,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:42,741 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:43,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:44,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:44,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:44,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:45,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:46,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:46,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:46,349 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:46,354 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:46,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:48,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:51,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:54,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:54,646 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:54,646 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:54,651 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:55,166 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:34:55,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:56,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:57,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:58,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:58,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:58,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:58,100 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:58,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:59,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:59,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:59,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:00,431 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:02,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:02,475 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:02,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:02,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:02,480 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:03,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:04,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:06,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:06,443 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:06,444 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:06,448 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:07,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:08,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:08,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:10,647 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:10,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:10,648 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:10,652 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:11,442 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:12,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:13,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:14,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:14,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:14,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:14,740 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:14,740 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:14,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:15,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:16,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:18,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:18,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:18,371 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:18,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:18,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:19,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:20,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:21,994 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:21,995 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:21,995 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:22,000 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:22,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:24,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:25,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:25,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:35:27,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:27,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:27,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:27,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:27,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:28,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:29,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:29,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:29,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:30,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:31,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:31,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:31,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:31,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:31,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:32,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:35,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:35,701 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:35,702 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:35,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:35,707 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:36,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:38,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:39,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:39,524 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:39,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:39,529 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:40,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:40,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:41,530 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:43,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:43,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:43,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:43,571 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:44,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:44,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:44,477 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:44,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:46,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:47,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:47,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:47,830 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:47,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:48,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:50,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:51,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:53,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:53,163 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:53,163 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:53,168 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:53,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:54,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:55,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:35:57,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:57,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:57,145 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:57,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:57,150 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:57,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:58,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:59,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:59,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:59,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:00,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:00,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:00,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:00,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:01,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:02,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:02,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:04,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:04,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:04,522 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:04,527 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:05,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:06,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:08,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:10,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:10,159 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:10,159 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:10,165 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:10,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:12,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:13,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:13,559 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:13,559 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:13,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:13,564 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:14,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:14,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:14,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:14,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:18,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:18,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:18,204 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:18,209 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:18,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:19,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:20,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:22,611 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:22,612 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:22,612 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:22,617 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:23,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:24,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:24,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:25,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:36:29,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:29,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:29,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:29,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:30,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:30,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:30,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:30,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:32,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:33,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:33,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:33,923 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:33,928 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:34,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:34,928 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:36,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:38,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:38,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:38,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:38,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:39,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:40,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:40,815 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:43,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:43,866 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:43,866 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:43,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:44,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:44,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:44,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:46,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:46,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:47,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:47,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:47,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:47,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:48,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:48,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:51,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:51,412 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:51,412 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:51,417 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:51,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:52,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:52,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:55,179 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:36:55,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:55,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:55,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:55,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:56,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:56,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:57,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:59,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:59,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:59,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:59,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:59,729 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:59,729 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:59,734 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:00,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:02,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:02,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:03,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:03,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:03,542 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:03,547 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:03,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:04,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:06,826 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:06,827 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:06,827 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:06,832 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:07,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:07,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:08,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:12,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:14,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:14,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:14,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:14,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:14,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:15,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:16,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:18,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:19,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:19,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:19,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:19,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:20,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:22,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:23,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:25,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:25,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:25,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:25,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:25,181 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:37:25,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:26,580 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:29,182 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:29,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:29,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:29,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:29,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:29,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:29,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:29,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:29,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:30,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:34,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:34,170 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:34,170 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:34,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:34,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:34,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:36,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:39,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:39,460 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:39,461 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:39,461 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:39,466 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:39,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:40,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:44,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:44,160 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:44,160 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:44,164 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:44,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:44,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:44,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:44,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:44,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:46,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:49,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:50,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:50,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:50,745 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:50,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:51,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:52,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:54,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:55,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:37:55,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:55,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:55,651 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:55,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:56,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:56,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:59,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:59,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:59,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:00,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:01,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:01,066 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:01,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:01,071 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:01,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:02,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:05,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:05,122 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:05,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:05,127 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:05,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:06,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:06,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:11,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:11,258 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:11,258 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:11,259 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:11,263 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:11,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:12,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:14,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:14,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:15,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:15,097 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:15,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:15,102 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:15,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:16,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:16,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:19,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:19,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:19,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:19,100 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:19,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:20,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:22,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:23,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:23,218 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:23,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:23,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:23,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:24,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:25,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:38:27,192 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:28,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:28,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:28,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:28,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:28,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:29,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:29,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:30,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:32,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:33,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:33,699 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:33,699 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:33,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:34,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:34,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:37,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:37,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:37,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:37,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:37,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:38,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:40,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:42,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:42,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:42,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:42,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:42,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:43,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:44,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:44,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:44,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:44,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:46,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:46,396 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:46,396 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:46,400 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:46,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:48,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:48,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:51,520 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:51,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:51,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:51,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:51,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:52,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:53,526 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:55,189 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:38:55,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:55,274 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:55,274 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:55,279 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:55,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:56,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:59,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:59,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:59,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:59,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:59,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:59,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:59,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:59,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:59,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:00,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:03,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:03,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:03,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:03,100 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:03,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:04,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:05,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:07,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:07,750 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:07,750 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:07,755 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:08,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:08,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:10,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:11,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:11,573 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:11,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:11,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:11,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:12,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:14,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:14,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:15,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:15,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:15,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:15,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:15,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:16,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:16,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:19,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:19,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:19,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:19,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:19,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:20,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:21,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:25,195 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:39:26,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:26,261 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:26,262 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:26,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:26,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:27,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:28,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:29,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:29,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:30,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:30,845 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:30,845 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:30,850 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:31,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:32,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:32,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:35,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:35,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:35,209 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:35,214 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:35,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:36,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:38,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:39,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:39,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:39,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:39,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:39,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:40,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:42,755 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:42,756 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:42,756 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:42,760 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:43,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:43,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:44,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:44,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:44,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:44,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:47,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:47,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:47,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:47,210 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:47,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:48,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:49,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:50,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:50,735 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:50,735 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:50,740 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:51,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:52,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:54,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:55,194 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:39:55,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:55,271 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:55,271 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:55,275 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:55,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:56,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:59,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:59,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:59,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:59,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:59,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:59,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:59,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:59,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:00,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:00,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:04,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:04,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:04,422 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:04,426 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:04,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:06,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:06,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:08,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:08,742 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:08,742 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:08,747 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:09,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:10,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:11,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:14,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:14,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:14,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:14,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:14,617 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:14,622 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:14,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:16,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:17,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:18,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:18,223 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:18,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:18,228 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:18,750 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:20,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:22,978 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:22,979 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:22,979 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:22,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:22,984 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:23,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:24,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:25,198 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:40:28,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:29,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:29,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:32,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:32,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:32,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:32,144 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:32,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:34,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:34,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:37,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:37,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:37,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:37,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:38,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:38,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:39,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:42,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:42,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:42,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:42,527 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:42,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:44,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:44,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:44,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:45,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:46,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:46,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:46,686 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:46,693 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:46,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:48,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:50,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:51,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:51,642 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:51,642 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:51,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:51,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:52,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:55,201 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:40:56,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:56,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:56,723 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:56,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:56,728 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:56,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:58,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:59,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:59,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:01,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:01,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:01,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:01,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:01,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:01,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:02,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:06,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:07,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:07,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:07,776 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:07,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:07,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:08,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:11,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:13,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:13,843 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:13,844 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:13,849 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:14,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:14,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:14,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:14,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:14,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:17,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:17,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:17,892 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:17,892 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:17,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:18,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:18,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:22,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:23,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:23,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:23,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:23,325 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:23,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:24,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:25,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:41:28,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:28,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:28,190 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:28,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:28,195 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:28,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:29,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:29,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:29,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:30,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:32,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:32,134 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:32,135 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:32,139 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:32,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:34,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:34,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:38,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:38,607 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:38,607 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:38,612 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:38,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:39,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:40,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:43,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:43,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:43,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:43,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:43,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:44,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:44,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:44,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:45,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:50,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:51,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:51,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:51,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:51,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:51,839 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:52,842 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:55,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:41:56,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:59,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:59,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:59,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:59,658 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:59,658 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:59,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:59,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:01,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:02,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:05,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:05,938 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:05,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:05,943 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:06,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:06,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:08,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:11,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:13,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:13,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:13,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:13,584 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:13,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:14,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:14,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:14,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:14,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:16,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:16,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:16,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:16,886 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:17,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:17,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:18,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:20,872 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:20,873 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:20,873 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:20,878 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:21,867 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:22,870 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:23,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:25,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:25,047 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:25,047 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:25,052 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:25,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:42:25,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:26,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:29,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:29,098 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:29,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:29,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:29,104 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:29,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:29,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:29,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:30,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:33,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:33,913 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:33,914 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:33,919 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:34,880 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:34,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:36,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:37,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:37,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:37,861 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:37,866 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:37,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:40,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:40,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:42,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:42,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:42,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:42,157 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:42,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:44,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:44,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:44,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:46,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:47,251 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:47,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:47,252 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:47,257 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:47,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:48,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:52,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:52,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:52,430 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:52,430 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:52,435 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:52,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:54,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:55,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:42:56,192 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:56,193 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:56,193 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:56,198 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:56,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:58,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:58,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:59,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:59,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:00,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:00,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:00,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:00,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:00,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:02,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:03,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:04,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:04,852 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:04,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:04,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:04,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:06,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:08,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:09,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:09,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:09,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:09,075 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:09,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:10,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:12,920 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:12,921 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:12,921 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:12,926 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:13,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:13,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:14,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:14,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:14,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:19,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:22,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:22,020 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:22,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:22,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:22,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:24,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:25,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:25,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:43:26,707 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:26,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:26,708 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:26,713 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:26,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:28,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:29,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:29,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:30,563 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:31,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:31,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:31,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:31,317 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:31,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:32,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:34,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:34,723 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:34,723 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:34,728 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:34,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:35,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:36,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:40,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:40,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:40,537 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:40,542 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:40,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:41,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:42,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:44,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:44,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:44,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:44,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:44,555 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:44,560 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:44,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:46,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:46,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:48,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:48,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:48,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:48,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:48,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:50,959 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:52,010 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:53,539 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:53,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:53,540 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:53,545 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:53,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:54,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:55,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:43:57,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:58,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:58,131 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:58,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:58,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:58,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:59,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:59,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:59,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:00,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:02,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:02,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:02,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:02,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:02,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:02,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:04,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:06,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:06,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:06,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:06,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:06,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:07,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:08,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:10,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:10,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:10,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:10,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:10,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:13,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:13,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:14,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:14,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:14,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:14,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:14,692 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:14,698 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:15,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:17,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:18,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:18,604 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:18,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:18,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:18,610 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:19,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:21,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:23,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:25,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:25,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:25,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:25,123 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:25,221 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:44:26,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:27,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:28,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:28,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:28,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:28,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:28,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:29,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:29,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:29,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:31,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:34,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:34,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:34,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:34,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:34,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:35,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:37,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:38,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:38,748 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:38,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:38,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:39,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:39,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:41,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:42,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:42,533 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:42,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:42,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:43,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:44,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:44,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:45,035 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:45,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:46,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:46,475 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:46,475 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:46,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:47,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:49,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:51,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:52,281 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:52,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:52,282 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:52,287 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:53,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:55,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:55,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:44:55,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:55,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:55,979 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:55,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:56,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:56,984 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:59,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:59,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:59,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:59,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:59,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:59,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:59,906 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:59,911 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:00,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:02,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:03,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:05,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:05,716 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:05,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:05,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:06,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:07,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:08,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:11,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:11,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:11,375 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:11,380 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:12,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:13,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:14,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:14,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:14,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:15,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:15,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:15,124 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:16,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:17,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:19,867 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:19,868 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:19,868 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:19,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:19,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:20,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:23,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:24,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:24,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:24,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:24,391 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:25,075 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:25,226 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:45:25,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:27,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:29,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:29,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:29,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:29,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:29,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:29,746 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:30,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:30,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:31,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:35,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:36,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:36,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:36,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:36,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:37,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:39,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:40,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:40,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:40,138 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:40,143 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:41,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:41,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:43,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:44,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:44,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:45,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:45,008 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:45,008 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:45,013 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:45,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:47,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:47,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:48,760 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:48,761 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:48,761 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:48,766 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:49,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:51,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:52,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:52,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:52,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:52,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:52,836 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:53,106 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:55,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:55,229 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:45:58,231 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:58,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:58,737 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:58,738 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:58,743 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:59,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:59,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:59,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:01,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:03,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:08,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:08,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:08,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:08,115 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:08,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:09,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:11,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:12,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:12,694 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:12,694 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:12,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:13,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:14,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:14,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:14,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:14,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:15,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:16,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:16,823 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:16,823 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:16,828 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:17,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:19,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:19,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:20,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:20,106 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:20,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:20,111 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:20,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:23,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:24,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:24,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:24,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:24,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:25,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:25,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:46:25,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:27,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:29,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:29,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:29,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:29,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:29,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:29,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:29,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:30,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:30,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:31,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:33,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:33,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:33,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:33,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:33,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:35,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:36,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:37,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:37,382 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:37,382 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:37,387 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:38,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:39,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:41,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:42,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:42,814 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:42,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:42,819 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:43,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:44,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:44,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:45,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:46,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:47,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:47,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:47,702 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:47,707 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:48,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:49,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:51,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:51,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:51,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:51,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:51,660 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:52,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:53,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:55,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:46:56,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:56,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:56,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:56,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:56,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:57,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:59,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:59,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:59,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:00,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:00,155 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:00,156 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:00,160 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:00,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:02,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:03,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:06,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:06,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:06,641 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:06,645 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:07,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:07,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:09,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:11,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:11,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:11,886 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:11,891 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:12,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:12,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:13,187 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:14,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:14,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:14,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:16,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:16,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:16,040 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:16,045 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:16,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:18,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:19,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:21,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:21,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:21,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:21,052 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:21,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:23,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:24,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:25,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:25,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:47:25,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:25,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:25,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:26,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:27,201 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:29,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:29,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:29,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:29,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:29,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:29,653 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:29,654 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:29,658 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:30,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:31,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:34,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:34,319 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:34,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:34,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:34,324 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:35,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:37,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:37,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:37,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:37,696 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:37,701 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:38,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:39,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:39,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:41,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:41,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:41,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:41,887 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:42,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:43,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:44,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:44,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:45,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:45,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:45,706 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:45,708 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:45,713 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:46,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:47,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:50,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:50,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:50,771 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:50,771 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:50,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:51,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:53,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:55,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:47:56,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:59,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:59,133 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:59,133 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:59,138 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:59,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:59,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:59,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:01,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:01,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:06,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:06,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:06,632 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:06,633 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:06,638 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:07,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:09,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:10,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:10,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:10,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:10,157 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:10,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:12,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:13,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:14,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:14,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:14,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:16,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:16,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:16,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:16,811 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:17,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:17,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:19,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:22,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:22,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:22,169 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:22,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:22,253 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:23,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:25,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:48:25,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:28,156 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:28,156 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:28,157 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:28,162 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:28,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:29,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:29,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:29,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:29,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:31,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:32,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:32,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:32,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:32,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:33,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:34,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:35,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:37,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:37,948 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:37,948 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:37,953 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:38,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:39,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:39,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:44,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:44,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:45,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:47,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:47,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:47,019 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:47,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:47,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:49,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:50,970 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:50,971 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:50,972 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:50,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:50,977 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:51,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:53,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:54,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:54,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:54,928 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:54,934 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:55,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:48:55,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:56,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:57,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:59,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:59,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:59,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:00,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:00,761 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:00,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:00,766 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:01,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:01,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:03,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:06,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:08,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:08,931 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:08,931 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:08,936 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:09,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:11,299 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:11,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:12,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:12,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:12,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:12,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:13,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:14,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:14,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:14,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:15,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:17,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:18,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:18,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:18,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:18,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:18,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:21,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:21,855 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:21,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:21,856 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:21,861 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:22,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:22,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:23,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:25,249 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:49:25,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:25,922 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:25,922 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:25,927 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:26,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:27,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:27,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:29,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:29,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:29,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:30,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:30,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:30,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:30,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:31,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:33,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:33,808 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:34,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:34,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:34,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:34,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:35,321 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:37,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:39,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:39,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:39,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:39,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:39,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:39,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:41,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:43,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:43,075 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:43,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:43,080 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:43,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:44,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:44,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:44,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:45,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:47,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:47,302 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:47,302 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:47,307 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:47,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:49,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:49,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:51,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:51,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:51,678 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:51,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:52,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:53,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:54,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:55,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:49:55,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:55,688 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:55,688 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:55,693 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:56,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:57,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:59,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:59,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:59,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:59,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:59,604 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:59,604 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:59,609 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:00,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:00,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:01,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:04,188 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:04,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:04,189 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:04,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:04,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:06,194 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:07,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:08,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:08,722 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:08,723 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:08,727 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:09,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:11,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:11,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:14,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:14,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:14,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:17,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:17,128 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:17,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:17,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:17,133 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:17,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:19,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:20,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:20,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:20,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:20,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:21,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:22,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:23,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:25,255 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:50:26,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:26,159 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:26,159 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:26,164 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:26,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:27,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:28,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:29,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:29,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:29,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:30,182 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:30,183 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:30,183 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:30,188 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:30,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:33,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:33,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:34,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:34,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:34,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:34,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:34,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:37,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:38,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:42,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:42,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:42,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:42,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:43,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:43,751 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:44,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:44,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:45,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:49,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:51,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:51,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:51,885 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:51,890 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:52,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:53,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:54,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:55,258 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:50:55,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:55,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:55,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:55,915 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:56,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:57,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:59,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:59,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:59,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:00,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:02,911 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:02,911 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:02,912 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:02,916 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:03,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:05,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:05,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:08,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:08,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:08,304 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:08,309 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:08,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:11,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:11,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:12,444 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:12,445 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:12,445 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:12,451 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:13,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:14,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:14,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:14,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:15,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:15,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:15,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:15,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:15,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:16,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:16,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:17,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:19,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:19,489 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:19,490 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:19,495 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:20,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:21,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:22,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:25,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:25,170 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:25,170 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:25,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:25,260 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:51:25,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:27,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:28,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:29,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:29,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:29,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:29,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:29,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:29,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:29,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:30,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:31,438 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:33,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:33,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:33,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:33,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:33,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:34,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:35,442 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:38,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:39,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:39,248 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:39,248 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:39,253 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:39,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:41,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:42,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:42,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:42,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:42,970 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:43,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:43,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:44,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:45,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:47,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:47,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:47,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:47,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:48,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:49,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:49,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:51,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:51,158 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:51,159 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:51,163 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:51,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:53,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:55,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:55,267 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:51:55,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:55,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:55,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:55,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:56,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:57,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:59,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:00,452 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:00,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:00,453 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:00,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:00,458 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:00,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:03,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:04,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:04,254 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:04,254 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:04,259 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:04,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:05,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:06,260 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:09,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:09,477 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:09,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:09,482 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:10,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:11,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:11,483 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:13,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:13,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:13,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:13,780 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:14,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:14,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:14,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:14,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:15,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:16,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:18,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:18,514 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:18,514 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:18,519 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:19,489 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:21,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:22,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:22,088 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:22,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:22,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:22,093 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:22,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:23,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:25,267 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:52:25,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:25,670 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:25,670 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:25,675 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:26,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:27,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:27,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:29,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:30,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:30,448 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:30,449 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:30,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:30,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:33,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:33,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:35,608 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:35,609 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:35,609 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:35,615 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:36,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:37,511 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:38,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:39,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:39,738 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:39,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:39,743 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:40,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:41,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:43,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:43,517 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:43,517 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:43,522 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:44,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:44,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:44,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:45,522 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:48,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:48,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:48,019 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:48,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:48,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:49,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:50,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:51,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:51,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:51,830 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:51,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:52,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:53,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:55,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:52:55,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:56,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:56,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:56,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:56,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:56,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:57,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:59,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:01,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:01,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:01,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:01,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:01,105 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:01,536 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:03,539 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:06,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:10,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:10,220 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:10,220 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:10,225 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:10,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:11,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:11,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:13,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:13,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:13,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:13,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:14,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:14,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:14,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:14,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:15,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:17,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:18,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:18,392 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:18,392 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:18,397 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:18,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:21,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:22,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:23,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:23,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:23,487 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:23,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:23,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:25,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:53:25,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:27,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:27,886 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:27,886 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:27,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:27,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:28,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:29,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:29,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:33,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:33,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:33,064 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:33,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:33,069 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:33,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:35,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:38,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:41,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:41,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:41,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:41,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:41,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:43,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:43,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:44,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:45,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:45,717 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:45,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:45,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:46,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:47,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:48,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:53,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:53,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:53,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:53,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:53,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:54,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:55,278 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:53:55,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:59,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:59,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:01,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:01,644 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:01,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:01,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:02,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:03,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:04,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:06,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:06,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:06,518 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:06,523 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:06,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:09,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:10,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:11,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:11,314 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:11,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:11,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:11,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:13,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:14,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:14,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:14,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:15,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:15,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:15,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:15,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:15,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:16,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:17,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:20,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:20,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:20,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:20,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:21,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:21,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:23,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:24,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:24,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:24,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:24,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:24,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:25,277 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:54:25,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:27,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:29,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:30,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:30,667 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:30,667 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:30,672 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:31,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:32,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:33,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:34,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:34,850 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:34,850 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:34,855 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:35,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:37,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:37,855 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:38,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:38,968 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:38,968 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:38,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:39,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:41,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:42,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:44,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:44,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:44,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:44,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:44,635 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:44,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:47,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:48,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:48,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:48,578 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:48,578 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:48,583 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:48,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:51,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:52,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:52,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:52,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:52,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:53,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:53,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:55,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:54:55,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:56,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:56,501 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:56,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:56,506 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:56,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:59,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:59,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:59,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:59,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:00,792 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:00,793 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:00,793 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:00,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:01,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:03,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:04,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:05,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:05,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:05,544 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:05,549 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:05,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:07,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:09,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:09,165 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:09,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:09,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:09,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:10,171 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:11,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:13,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:13,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:13,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:13,610 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:13,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:14,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:14,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:14,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:15,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:16,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:17,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:17,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:17,830 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:17,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:18,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:19,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:21,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:22,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:22,464 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:22,465 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:22,470 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:22,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:23,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:25,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:55:26,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:26,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:26,726 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:26,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:27,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:27,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:29,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:29,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:29,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:30,450 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:30,451 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:30,451 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:30,456 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:30,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:31,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:33,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:35,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:35,670 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:35,670 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:35,675 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:35,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:37,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:38,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:40,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:40,662 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:40,662 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:40,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:40,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:43,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:44,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:44,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:44,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:45,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:45,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:45,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:45,269 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:45,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:47,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:50,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:50,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:50,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:50,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:50,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:50,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:53,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:55,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:55:56,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:56,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:56,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:56,622 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:56,627 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:56,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:59,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:59,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:59,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:59,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:00,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:00,328 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:00,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:00,333 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:00,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:01,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:01,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:04,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:04,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:04,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:04,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:04,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:06,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:07,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:10,250 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:10,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:10,251 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:10,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:10,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:11,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:12,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:14,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:14,584 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:14,584 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:14,589 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:14,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:14,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:14,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:14,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:17,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:18,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:18,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:18,562 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:18,563 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:18,567 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:18,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:21,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:23,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:25,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:56:26,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:26,932 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:26,933 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:26,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:27,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:28,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:29,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:29,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:29,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:29,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:33,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:33,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:33,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:33,094 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:33,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:34,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:35,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:37,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:37,845 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:37,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:37,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:38,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:39,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:39,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:41,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:41,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:41,224 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:41,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:41,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:43,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:44,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:44,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:44,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:46,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:46,200 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:46,200 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:46,205 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:46,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:47,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:50,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:51,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:51,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:51,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:51,119 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:51,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:53,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:55,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:56:55,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:56,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:56,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:56,588 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:56,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:56,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:57,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:59,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:59,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:59,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:01,025 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:01,026 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:01,026 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:01,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:01,031 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:01,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:03,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:06,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:09,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:09,535 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:09,535 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:09,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:09,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:11,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:11,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:13,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:13,900 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:13,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:13,906 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:14,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:14,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:14,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:14,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:15,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:17,015 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:18,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:18,650 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:18,650 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:18,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:18,789 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:21,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:22,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:24,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:24,787 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:24,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:24,792 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:24,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:25,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:57:27,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:28,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:29,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:29,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:29,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:30,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:30,266 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:30,267 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:30,271 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:30,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:31,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:33,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:38,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:38,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:38,705 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:38,705 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:38,710 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:38,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:41,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:43,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:43,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:43,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:43,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:43,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:44,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:44,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:44,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:45,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:49,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:49,495 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:49,496 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:49,496 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:49,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:49,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:51,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:54,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:54,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:54,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:54,440 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:54,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:55,297 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:57:55,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:55,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:59,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:59,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:59,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:00,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:00,578 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:00,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:00,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:00,583 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:00,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:01,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:04,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:04,581 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:04,581 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:04,586 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:04,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:05,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:05,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:08,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:08,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:08,947 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:08,952 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:09,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:10,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:11,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:14,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:14,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:14,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:15,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:15,023 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:15,023 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:15,028 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:15,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:16,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:17,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:19,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:19,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:19,666 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:19,671 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:19,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:21,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:21,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:23,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:23,154 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:23,155 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:23,160 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:23,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:25,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:58:25,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:27,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:28,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:28,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:28,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:28,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:28,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:29,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:29,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:29,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:29,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:32,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:32,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:32,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:32,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:32,513 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:32,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:33,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:37,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:40,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:40,867 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:40,867 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:40,872 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:41,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:42,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:43,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:44,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:44,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:44,696 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:44,701 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:44,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:44,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:45,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:48,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:48,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:48,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:48,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:48,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:48,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:49,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:53,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:55,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:58:57,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:57,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:57,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:57,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:57,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:59,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:59,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:59,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:59,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:59,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:01,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:01,198 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:01,198 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:01,203 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:01,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:03,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:04,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:06,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:06,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:06,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:06,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:06,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:07,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:09,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:10,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:10,519 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:10,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:10,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:10,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:11,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:14,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:14,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:14,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:15,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:15,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:15,994 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:15,994 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:15,999 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:16,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:17,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:20,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:23,652 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:23,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:23,653 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:23,658 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:23,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:25,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:59:25,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:25,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:29,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:29,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:29,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:31,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:32,751 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:32,751 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:32,752 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:32,757 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:32,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:35,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:36,675 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:36,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:36,676 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:36,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:36,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:36,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:37,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:40,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:40,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:40,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:40,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:40,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:41,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:42,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:43,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:43,964 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:43,965 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:43,969 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:44,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:44,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:45,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:48,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:48,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:48,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:48,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:48,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:48,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:49,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:53,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:54,994 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:54,995 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:54,995 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:55,000 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:55,308 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:59:55,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:57,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:59,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:59,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:59,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:59,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:59,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:59,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:59,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:59,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:01,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:03,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:03,060 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:03,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:03,065 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:03,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:04,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:05,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:07,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:07,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:07,651 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:07,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:07,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:09,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:09,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:12,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:12,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:12,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:12,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:12,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:13,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:14,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:14,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:14,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:16,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:16,654 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:16,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:16,660 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:16,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:17,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:20,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:21,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:21,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:21,190 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:21,195 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:21,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:23,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:25,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:00:26,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:26,047 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:26,047 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:26,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:26,052 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:26,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:27,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:29,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:29,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:29,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:30,895 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:30,895 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:30,896 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:30,901 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:30,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:31,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:33,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:34,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:34,538 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:34,538 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:34,543 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:34,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:35,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:37,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:38,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:38,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:38,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:38,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:38,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:41,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:42,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:43,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:43,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:43,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:43,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:43,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:44,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:44,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:45,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:47,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:47,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:47,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:47,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:47,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:48,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:50,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:52,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:52,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:52,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:52,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:53,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:54,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:54,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:55,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:00:58,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:58,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:58,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:58,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:59,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:59,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:59,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:59,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:00,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:02,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:03,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:03,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:03,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:03,100 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:04,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:05,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:06,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:07,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:07,399 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:07,400 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:07,404 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:08,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:10,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:10,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:12,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:12,171 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:12,171 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:12,176 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:13,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:14,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:14,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:14,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:14,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:16,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:16,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:16,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:16,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:16,069 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:17,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:18,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:20,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:20,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:20,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:20,092 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:21,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:21,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:22,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:25,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:01:25,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:25,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:25,375 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:25,380 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:26,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:26,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:28,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:29,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:29,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:29,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:29,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:29,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:29,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:29,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:30,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:32,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:32,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:33,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:33,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:33,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:33,359 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:34,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:36,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:37,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:38,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:38,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:38,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:38,580 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:39,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:40,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:41,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:41,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:41,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:41,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:42,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:42,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:44,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:44,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:44,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:44,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:46,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:46,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:46,029 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:46,034 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:46,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:48,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:48,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:49,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:49,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:49,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:49,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:50,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:52,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:53,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:54,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:54,302 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:54,302 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:54,307 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:55,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:55,321 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:01:56,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:59,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:59,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:59,398 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:59,398 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:59,403 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:59,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:59,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:59,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:00,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:02,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:05,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:05,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:05,691 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:05,692 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:05,697 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:06,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:08,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:09,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:09,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:09,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:09,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:10,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:10,835 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:12,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:14,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:14,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:14,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:16,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:19,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:19,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:19,764 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:19,769 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:20,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:21,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:22,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:24,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:24,387 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:24,387 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:24,392 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:25,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:25,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:02:26,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:27,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:29,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:29,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:29,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:31,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:31,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:31,348 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:31,353 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:32,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:32,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:34,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:36,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:36,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:36,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:36,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:37,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:37,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:38,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:41,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:41,497 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:41,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:41,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:42,113 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:43,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:44,116 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:44,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:44,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:44,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:46,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:46,417 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:46,418 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:46,423 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:47,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:48,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:49,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:51,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:51,137 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:51,138 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:51,143 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:52,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:54,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:55,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:55,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:02:57,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:57,230 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:57,230 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:57,236 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:58,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:59,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:59,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:59,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:00,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:01,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:01,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:01,444 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:01,444 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:01,449 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:02,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:04,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:05,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:05,649 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:05,650 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:05,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:06,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:06,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:08,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:10,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:10,530 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:10,530 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:10,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:11,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:12,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:12,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:14,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:14,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:14,772 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:14,777 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:14,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:14,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:14,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:15,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:16,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:18,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:19,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:19,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:19,440 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:19,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:20,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:22,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:23,481 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:24,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:24,050 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:24,051 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:24,056 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:24,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:25,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:03:26,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:28,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:28,278 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:28,278 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:28,283 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:29,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:29,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:29,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:29,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:29,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:30,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:35,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:35,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:35,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:35,900 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:35,905 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:36,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:38,173 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:40,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:42,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:42,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:42,849 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:42,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:43,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:44,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:44,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:44,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:46,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:46,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:46,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:46,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:46,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:47,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:48,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:51,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:51,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:51,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:51,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:51,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:52,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:54,189 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:55,330 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:03:57,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:57,520 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:57,520 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:57,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:57,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:58,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:59,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:59,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:59,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:00,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:02,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:02,485 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:02,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:02,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:02,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:03,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:04,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:06,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:06,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:06,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:06,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:06,201 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:08,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:08,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:10,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:10,265 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:10,265 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:10,270 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:11,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:12,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:13,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:14,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:14,473 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:14,474 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:14,479 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:14,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:14,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:14,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:15,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:16,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:19,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:19,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:19,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:19,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:19,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:20,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:22,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:23,664 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:23,665 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:23,665 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:23,670 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:24,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:24,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:25,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:04:26,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:29,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:29,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:29,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:30,042 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:32,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:32,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:32,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:32,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:33,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:34,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:35,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:37,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:37,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:37,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:37,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:38,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:40,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:41,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:41,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:41,504 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:41,504 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:41,509 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:42,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:44,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:44,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:45,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:45,101 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:45,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:45,106 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:45,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:46,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:47,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:50,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:50,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:50,039 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:50,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:50,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:52,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:53,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:53,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:53,472 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:53,472 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:53,478 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:54,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:55,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:04:56,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:58,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:59,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:59,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:59,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:59,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:59,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:00,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:02,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:04,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:05,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:05,296 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:05,296 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:05,301 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:06,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:08,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:09,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:11,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:11,183 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:11,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:11,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:11,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:14,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:14,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:14,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:14,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:15,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:20,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:20,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:20,522 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:20,522 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:20,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:21,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:22,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:23,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:23,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:23,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:23,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:24,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:25,342 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:05:25,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:26,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:27,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:27,847 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:27,847 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:27,852 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:28,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:29,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:29,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:29,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:30,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:31,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:33,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:33,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:33,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:33,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:34,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:36,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:36,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:37,727 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:37,728 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:37,729 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:37,734 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:38,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:40,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:41,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:41,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:41,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:41,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:42,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:42,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:44,299 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:44,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:45,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:45,729 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:45,730 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:45,734 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:46,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:47,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:48,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:49,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:49,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:49,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:49,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:50,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:52,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:53,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:54,284 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:54,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:54,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:54,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:54,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:55,341 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:05:56,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:57,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:57,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:57,660 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:57,665 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:58,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:59,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:59,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:00,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:02,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:02,271 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:02,271 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:02,276 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:02,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:04,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:05,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:07,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:07,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:07,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:07,090 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:07,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:08,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:11,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:13,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:13,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:13,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:13,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:14,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:14,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:14,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:14,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:16,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:17,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:17,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:17,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:17,468 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:17,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:18,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:20,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:22,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:25,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:06:25,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:25,750 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:25,750 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:25,755 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:26,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:27,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:28,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:29,336 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:29,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:29,342 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:29,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:29,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:29,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:29,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:32,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:33,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:33,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:33,310 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:33,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:33,315 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:33,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:36,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:38,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:39,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:39,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:39,034 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:39,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:39,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:40,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:44,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:44,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:44,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:47,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:47,020 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:47,020 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:47,025 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:47,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:48,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:50,025 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:51,755 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:51,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:51,756 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:51,761 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:52,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:54,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:55,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:06:55,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:55,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:55,596 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:55,597 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:55,602 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:56,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:58,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:59,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:00,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:00,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:00,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:00,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:00,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:01,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:02,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:05,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:06,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:06,109 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:06,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:06,115 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:06,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:08,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:11,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:11,608 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:11,608 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:11,609 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:11,614 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:12,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:14,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:14,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:14,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:14,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:15,616 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:15,616 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:15,617 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:15,621 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:16,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:16,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:18,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:19,800 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:19,800 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:19,801 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:19,806 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:20,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:21,806 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:22,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:23,982 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:23,982 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:23,983 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:23,987 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:24,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:25,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:07:26,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:27,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:29,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:29,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:29,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:30,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:30,434 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:30,434 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:30,439 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:31,431 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:32,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:32,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:37,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:38,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:38,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:38,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:38,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:39,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:40,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:42,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:44,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:44,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:44,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:44,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:44,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:44,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:44,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:44,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:46,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:48,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:48,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:48,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:48,224 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:48,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:48,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:50,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:53,230 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:54,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:54,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:54,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:54,105 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:54,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:55,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:07:56,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:58,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:59,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:00,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:00,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:00,823 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:00,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:01,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:02,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:03,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:04,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:04,477 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:04,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:04,482 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:05,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:06,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:08,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:08,574 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:08,574 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:08,579 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:09,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:09,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:10,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:13,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:13,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:13,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:13,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:13,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:14,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:14,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:14,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:14,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:16,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:16,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:16,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:16,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:17,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:18,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:20,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:22,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:22,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:22,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:22,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:22,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:24,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:25,355 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:08:26,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:29,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:29,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:29,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:31,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:31,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:31,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:31,110 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:31,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:32,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:32,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:35,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:35,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:35,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:35,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:35,522 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:36,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:37,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:40,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:40,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:40,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:40,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:41,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:42,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:42,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:44,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:44,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:44,320 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:44,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:44,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:44,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:46,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:48,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:48,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:48,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:48,253 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:48,258 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:48,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:50,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:52,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:52,849 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:52,849 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:52,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:53,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:53,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:54,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:55,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:08:56,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:56,165 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:56,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:56,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:56,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:58,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:59,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:59,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:59,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:00,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:00,520 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:00,520 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:00,525 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:00,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:02,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:04,526 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:06,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:06,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:06,348 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:06,353 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:06,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:08,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:10,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:10,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:10,576 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:10,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:10,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:11,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:12,578 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:14,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:14,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:14,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:15,842 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:15,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:15,843 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:15,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:15,848 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:16,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:18,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:20,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:20,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:20,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:20,976 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:20,981 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:21,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:22,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:25,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:25,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:25,224 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:25,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:25,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:09:25,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:26,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:26,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:29,057 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:29,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:29,065 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:29,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:29,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:29,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:29,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:30,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:32,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:32,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:32,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:32,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:32,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:33,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:34,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:37,454 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:37,455 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:37,456 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:37,456 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:37,460 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:37,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:38,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:41,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:41,621 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:41,621 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:41,626 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:42,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:42,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:44,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:44,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:48,010 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:50,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:50,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:50,003 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:50,008 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:50,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:52,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:54,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:54,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:54,573 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:54,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:54,578 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:54,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:55,365 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:09:56,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:58,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:58,199 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:58,199 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:58,204 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:58,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:59,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:59,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:59,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:59,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:00,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:02,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:02,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:02,169 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:02,174 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:02,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:04,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:05,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:06,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:06,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:06,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:06,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:06,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:08,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:10,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:10,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:10,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:10,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:10,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:10,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:12,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:14,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:14,166 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:14,166 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:14,171 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:14,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:14,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:14,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:14,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:16,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:16,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:19,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:19,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:19,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:19,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:20,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:21,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:22,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:23,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:23,495 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:23,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:23,500 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:23,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:24,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:25,364 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:10:27,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:29,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:29,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:32,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:32,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:32,721 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:32,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:32,726 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:33,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:34,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:36,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:36,357 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:36,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:36,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:36,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:38,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:38,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:40,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:40,365 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:40,365 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:40,370 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:40,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:42,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:43,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:44,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:44,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:44,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:44,230 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:44,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:44,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:46,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:49,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:49,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:49,563 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:49,564 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:49,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:49,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:52,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:54,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:54,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:54,360 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:54,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:54,365 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:54,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:55,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:10:56,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:59,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:59,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:59,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:59,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:00,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:00,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:00,656 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:00,661 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:00,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:02,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:04,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:04,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:04,699 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:04,699 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:04,704 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:04,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:06,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:08,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:08,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:08,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:08,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:08,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:10,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:10,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:14,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:14,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:14,727 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:14,732 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:14,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:14,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:14,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:14,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:16,025 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:16,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:18,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:18,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:18,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:18,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:19,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:20,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:21,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:25,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:11:26,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:26,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:26,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:26,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:26,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:27,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:28,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:29,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:29,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:31,810 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:31,811 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:31,811 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:31,816 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:32,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:32,816 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:34,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:37,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:38,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:38,140 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:38,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:38,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:38,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:40,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:43,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:44,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:44,014 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:44,014 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:44,019 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:44,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:44,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:44,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:46,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:48,764 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:48,765 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:48,765 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:48,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:48,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:48,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:50,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:53,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:53,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:53,624 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:53,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:53,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:54,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:55,376 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:11:56,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:57,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:57,177 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:57,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:57,182 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:57,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:58,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:59,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:59,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:59,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:00,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:02,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:02,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:02,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:02,172 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:02,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:04,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:05,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:07,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:07,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:07,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:07,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:07,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:08,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:10,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:14,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:14,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:14,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:15,998 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:16,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:16,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:16,138 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:16,143 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:16,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:18,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:20,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:20,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:20,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:20,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:20,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:21,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:22,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:25,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:12:25,855 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:25,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:25,856 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:25,863 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:25,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:26,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:28,867 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:29,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:30,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:30,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:30,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:30,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:30,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:32,811 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:32,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:34,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:34,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:34,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:34,729 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:34,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:36,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:38,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:38,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:38,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:38,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:38,117 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:38,880 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:40,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:42,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:42,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:42,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:42,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:42,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:43,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:44,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:44,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:44,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:44,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:46,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:46,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:46,169 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:46,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:46,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:48,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:49,176 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:50,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:50,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:50,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:50,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:50,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:52,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:54,230 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:54,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:54,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:54,471 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:54,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:54,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:55,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:12:56,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:58,450 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:58,451 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:58,452 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:58,458 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:58,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:59,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:59,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:59,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:59,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:00,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:04,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:04,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:04,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:04,315 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:04,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:05,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:06,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:08,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:08,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:08,672 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:08,677 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:08,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:10,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:10,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:12,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:12,514 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:12,514 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:12,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:12,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:14,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:14,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:14,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:14,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:16,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:17,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:17,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:17,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:17,173 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:17,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:18,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:20,968 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:20,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:20,969 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:20,974 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:21,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:21,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:22,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:25,382 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:13:26,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:26,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:26,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:26,796 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:26,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:27,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:28,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:29,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:31,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:31,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:31,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:31,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:31,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:32,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:33,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:35,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:35,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:35,375 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:35,381 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:35,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:36,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:38,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:43,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:43,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:43,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:43,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:43,579 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:43,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:44,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:44,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:44,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:44,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:48,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:48,057 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:48,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:48,063 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:48,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:49,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:50,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:52,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:52,384 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:52,384 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:52,389 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:52,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:54,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:54,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:55,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:13:56,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:56,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:56,347 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:56,352 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:56,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:58,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:59,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:59,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:00,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:01,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:01,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:01,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:01,998 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:02,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:05,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:05,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:06,563 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:06,564 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:06,564 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:06,569 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:07,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:09,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:10,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:10,565 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:10,566 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:10,571 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:11,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:11,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:13,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:14,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:14,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:14,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:17,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:18,870 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:18,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:18,871 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:18,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:19,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:21,033 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:22,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:23,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:23,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:23,954 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:23,963 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:24,035 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:25,393 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:14:27,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:28,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:28,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:28,167 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:28,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:28,174 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:29,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:29,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:31,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:32,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:32,734 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:32,735 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:32,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:33,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:33,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:35,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:37,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:37,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:37,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:37,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:38,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:39,075 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:39,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:41,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:41,715 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:41,715 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:41,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:42,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:43,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:44,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:44,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:44,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:44,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:46,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:46,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:46,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:46,707 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:47,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:49,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:50,487 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:50,488 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:50,488 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:50,488 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:50,496 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:51,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:52,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:54,251 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:54,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:54,252 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:54,262 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:55,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:55,391 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:14:56,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:56,392 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:58,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:58,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:58,217 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:58,222 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:59,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:59,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:00,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:02,022 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:03,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:03,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:03,765 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:03,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:04,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:04,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:07,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:10,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:10,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:10,031 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:10,038 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:10,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:12,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:13,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:13,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:13,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:13,861 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:13,869 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:14,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:14,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:14,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:14,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:16,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:17,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:17,639 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:17,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:17,646 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:18,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:18,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:18,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:21,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:21,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:21,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:21,600 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:22,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:22,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:24,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:25,399 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:15:26,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:26,559 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:26,559 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:26,570 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:27,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:28,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:29,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:30,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:32,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:32,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:32,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:32,019 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:32,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:34,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:35,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:36,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:36,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:36,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:36,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:36,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:38,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:39,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:39,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:39,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:39,817 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:40,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:40,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:40,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:44,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:44,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:44,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:46,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:48,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:48,230 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:48,230 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:48,238 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:49,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:50,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:51,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:54,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:54,934 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:54,935 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:54,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:55,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:55,396 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:15:56,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:56,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:59,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:02,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:03,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:03,602 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:03,603 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:03,612 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:04,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:04,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:07,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:07,466 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:07,466 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:07,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:07,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:08,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:08,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:11,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:11,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:11,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:11,474 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:12,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:12,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:12,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:14,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:14,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:14,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:15,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:15,523 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:15,523 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:15,532 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:16,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:16,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:17,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:22,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:24,171 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:24,172 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:24,172 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:24,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:24,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:25,403 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:16:26,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:28,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:29,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:29,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:30,022 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:30,022 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:30,032 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:30,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:32,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:33,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:33,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:33,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:33,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:33,684 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:34,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:34,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:38,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:38,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:38,421 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:38,431 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:39,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:39,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:40,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:44,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:44,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:44,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:44,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:46,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:46,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:46,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:46,960 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:47,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:48,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:49,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:51,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:51,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:51,408 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:51,417 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:52,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:52,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:55,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:16:55,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:57,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:57,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:57,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:57,117 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:57,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:58,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:59,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:59,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:01,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:01,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:01,062 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:01,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:01,073 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:01,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:02,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:05,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:05,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:05,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:05,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:06,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:06,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:06,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:11,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:11,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:11,803 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:11,803 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:11,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:12,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:12,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:14,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:14,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:14,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:16,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:16,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:16,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:16,727 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:17,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:17,727 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:18,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:20,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:20,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:20,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:20,625 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:21,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:22,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:23,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:24,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:24,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:24,421 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:24,429 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:25,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:17:25,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:26,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:29,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:29,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:29,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:32,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:32,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:32,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:32,790 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:33,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:34,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:34,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:36,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:36,547 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:36,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:36,555 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:37,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:38,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:40,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:40,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:40,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:40,997 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:41,005 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:41,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:42,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:44,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:44,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:44,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:44,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:44,792 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:44,792 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:45,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:46,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:46,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:48,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:48,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:48,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:48,846 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:49,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:50,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:51,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:53,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:53,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:53,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:53,756 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:54,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:54,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:55,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:17:57,408 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:57,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:57,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:57,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:57,783 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:58,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:58,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:59,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:59,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:59,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:02,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:02,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:02,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:02,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:02,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:03,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:04,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:06,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:06,418 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:06,418 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:06,424 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:06,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:08,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:08,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:12,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:12,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:12,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:12,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:12,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:14,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:14,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:14,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:14,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:14,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:17,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:17,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:17,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:17,913 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:18,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:18,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:19,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:22,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:22,340 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:22,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:22,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:22,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:24,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:25,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:25,410 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:18:26,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:26,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:26,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:26,152 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:26,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:28,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:29,787 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:29,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:29,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:29,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:29,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:29,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:30,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:30,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:31,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:35,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:35,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:35,217 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:35,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:35,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:36,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:36,539 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:41,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:41,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:41,490 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:41,490 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:41,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:41,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:42,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:44,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:44,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:44,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:45,478 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:45,479 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:45,479 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:45,485 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:45,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:46,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:46,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:51,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:52,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:52,099 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:52,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:52,105 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:52,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:54,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:55,417 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:18:57,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:57,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:57,902 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:57,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:57,910 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:58,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:58,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:59,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:59,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:59,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:01,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:01,763 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:01,763 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:01,770 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:02,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:02,580 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:02,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:07,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:07,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:07,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:07,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:07,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:08,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:08,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:12,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:12,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:12,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:12,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:12,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:13,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:14,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:14,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:14,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:14,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:18,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:18,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:18,630 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:18,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:18,635 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:19,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:20,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:23,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:24,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:24,926 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:24,926 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:24,931 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:25,415 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:19:25,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:26,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:29,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:29,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:29,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:29,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:33,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:33,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:33,413 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:33,420 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:33,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:34,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:34,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:37,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:37,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:37,353 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:37,358 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:37,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:38,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:40,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:43,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:43,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:43,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:43,113 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:43,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:44,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:44,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:44,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:44,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:46,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:49,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:49,257 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:49,258 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:49,263 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:49,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:50,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:51,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:54,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:54,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:54,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:54,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:54,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:55,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:19:56,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:56,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:58,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:58,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:58,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:58,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:59,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:59,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:59,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:59,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:00,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:02,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:02,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:02,271 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:02,271 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:02,276 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:02,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:04,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:05,966 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:05,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:05,967 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:05,972 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:06,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:06,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:07,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:10,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:10,704 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:10,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:10,709 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:11,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:12,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:13,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:14,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:14,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:14,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:14,910 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:15,073 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:15,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:15,078 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:15,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:16,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:19,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:20,468 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:20,469 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:20,469 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:20,474 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:20,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:22,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:24,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:25,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:20:28,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:28,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:28,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:28,895 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:29,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:29,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:29,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:29,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:30,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:30,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:33,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:33,208 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:33,209 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:33,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:33,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:34,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:35,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:37,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:37,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:37,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:37,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:37,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:38,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:40,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:41,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:41,229 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:41,229 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:41,234 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:41,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:42,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:44,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:44,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:44,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:45,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:45,289 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:45,289 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:45,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:45,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:46,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:46,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:51,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:53,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:53,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:53,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:53,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:53,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:54,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:55,427 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:20:56,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:57,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:57,847 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:57,847 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:57,852 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:58,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:58,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:59,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:59,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:59,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:01,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:01,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:01,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:01,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:01,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:02,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:02,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:06,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:06,434 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:06,434 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:06,439 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:06,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:07,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:08,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:11,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:11,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:11,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:11,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:11,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:12,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:13,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:14,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:14,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:14,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:15,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:15,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:15,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:15,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:15,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:16,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:18,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:19,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:19,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:19,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:19,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:19,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:20,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:24,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:25,427 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:21:25,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:25,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:25,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:25,958 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:26,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:26,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:29,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:29,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:29,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:30,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:30,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:30,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:30,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:30,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:30,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:35,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:38,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:38,607 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:38,607 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:38,612 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:38,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:40,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:40,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:43,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:43,088 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:43,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:43,093 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:43,750 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:44,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:44,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:44,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:44,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:46,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:47,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:47,040 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:47,041 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:47,045 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:47,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:48,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:51,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:51,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:51,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:51,771 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:51,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:52,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:52,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:55,430 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:21:56,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:56,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:56,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:56,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:56,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:56,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:58,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:59,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:59,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:59,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:00,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:00,613 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:00,613 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:00,618 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:00,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:01,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:02,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:06,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:06,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:06,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:06,093 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:06,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:06,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:07,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:10,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:10,258 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:10,258 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:10,263 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:10,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:10,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:12,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:14,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:14,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:14,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:14,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:14,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:14,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:14,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:14,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:16,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:18,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:19,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:19,667 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:19,668 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:19,673 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:19,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:20,790 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:23,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:25,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:25,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:25,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:25,394 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:25,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:22:25,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:26,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:29,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:29,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:29,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:29,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:33,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:33,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:33,836 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:33,840 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:34,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:34,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:34,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:37,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:37,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:37,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:37,665 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:37,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:38,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:40,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:41,752 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:41,753 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:41,753 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:41,758 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:41,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:42,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:44,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:44,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:44,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:45,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:45,444 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:45,444 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:45,449 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:45,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:46,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:46,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:49,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:49,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:49,018 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:49,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:49,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:50,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:52,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:53,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:53,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:53,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:53,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:53,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:54,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:55,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:22:57,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:57,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:57,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:57,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:57,317 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:57,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:58,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:59,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:59,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:59,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:01,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:01,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:01,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:01,534 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:01,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:02,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:02,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:05,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:05,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:05,695 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:05,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:05,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:06,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:07,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:09,748 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:09,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:09,749 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:09,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:09,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:10,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:12,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:13,994 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:13,995 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:13,995 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:14,000 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:14,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:14,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:14,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:14,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:14,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:18,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:19,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:19,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:19,350 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:19,355 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:19,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:20,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:23,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:23,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:23,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:23,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:23,195 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:23,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:24,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:25,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:23:28,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:29,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:29,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:29,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:29,006 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:29,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:29,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:29,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:29,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:30,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:33,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:33,844 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:33,845 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:33,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:33,849 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:33,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:34,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:38,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:39,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:39,218 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:39,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:39,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:39,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:40,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:42,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:42,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:42,956 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:42,961 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:43,954 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:43,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:44,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:44,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:44,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:44,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:46,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:46,365 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:46,366 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:46,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:46,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:46,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:49,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:51,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:51,048 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:51,048 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:51,054 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:51,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:52,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:54,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:54,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:54,965 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:54,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:54,970 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:55,441 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:23:55,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:56,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:59,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:59,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:59,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:59,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:00,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:00,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:00,095 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:00,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:00,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:04,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:04,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:04,991 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:04,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:05,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:05,997 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:06,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:10,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:10,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:10,772 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:10,777 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:10,982 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:11,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:12,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:14,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:14,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:14,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:14,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:15,098 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:15,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:15,103 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:15,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:16,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:17,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:18,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:18,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:18,380 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:18,385 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:18,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:18,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:22,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:22,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:22,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:22,925 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:22,930 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:22,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:24,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:25,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:24:27,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:28,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:28,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:28,176 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:28,181 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:29,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:29,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:29,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:29,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:29,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:32,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:32,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:32,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:32,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:32,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:33,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:35,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:37,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:39,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:39,109 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:39,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:39,115 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:40,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:41,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:42,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:42,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:42,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:42,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:42,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:43,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:45,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:45,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:47,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:47,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:47,796 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:47,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:47,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:48,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:49,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:51,835 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:51,836 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:51,836 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:51,841 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:52,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:52,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:53,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:55,448 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:24:58,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:00,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:00,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:00,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:00,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:00,329 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:00,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:00,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:01,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:01,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:04,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:04,982 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:04,983 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:04,983 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:04,988 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:05,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:07,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:09,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:09,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:09,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:09,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:10,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:10,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:11,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:15,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:15,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:15,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:17,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:17,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:17,925 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:17,930 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:18,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:19,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:20,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:25,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:25:26,454 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:26,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:26,522 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:26,522 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:26,527 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:27,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:29,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:30,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:30,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:30,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:30,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:30,354 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:30,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:30,359 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:31,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:31,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:32,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:34,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:34,002 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:34,002 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:34,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:34,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:35,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:38,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:38,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:38,155 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:38,155 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:38,160 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:39,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:39,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:43,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:45,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:46,526 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:46,526 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:46,527 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:46,532 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:47,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:48,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:49,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:50,136 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:50,137 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:50,137 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:50,142 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:51,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:51,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:54,142 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:55,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:55,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:55,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:55,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:55,452 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:25:56,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:57,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:59,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:59,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:59,217 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:59,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:59,222 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:00,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:00,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:00,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:00,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:01,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:03,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:03,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:03,928 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:03,932 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:04,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:04,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:05,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:08,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:08,981 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:08,981 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:08,986 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:09,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:09,986 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:11,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:12,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:12,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:12,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:12,571 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:13,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:13,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:15,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:17,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:17,346 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:17,347 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:17,352 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:18,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:19,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:20,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:21,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:21,834 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:21,834 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:21,839 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:22,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:23,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:25,456 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:26:25,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:26,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:26,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:26,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:26,109 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:26,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:27,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:30,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:30,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:30,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:31,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:32,868 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:32,869 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:32,869 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:32,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:33,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:35,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:36,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:37,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:37,586 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:37,586 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:37,591 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:38,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:39,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:42,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:44,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:44,456 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:44,456 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:44,461 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:45,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:45,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:45,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:48,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:49,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:49,058 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:49,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:49,063 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:49,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:51,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:54,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:54,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:54,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:54,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:54,113 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:54,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:55,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:55,458 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:26:59,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:00,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:00,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:00,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:02,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:02,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:02,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:02,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:03,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:04,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:05,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:07,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:07,503 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:07,503 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:07,508 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:08,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:09,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:10,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:11,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:11,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:11,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:11,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:12,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:13,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:15,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:16,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:17,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:17,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:17,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:17,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:17,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:19,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:20,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:20,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:20,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:20,879 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:21,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:21,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:23,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:25,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:25,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:25,239 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:25,244 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:25,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:27:26,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:27,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:27,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:29,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:29,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:29,192 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:29,198 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:30,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:30,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:30,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:30,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:31,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:33,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:33,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:33,443 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:33,444 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:33,448 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:34,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:35,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:37,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:37,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:37,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:37,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:38,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:39,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:39,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:41,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:41,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:41,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:41,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:42,199 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:43,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:45,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:46,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:46,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:46,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:46,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:46,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:47,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:50,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:50,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:50,466 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:50,466 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:50,471 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:51,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:51,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:55,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:27:55,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:55,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:55,484 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:55,484 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:55,489 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:56,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:57,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:00,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:00,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:00,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:01,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:02,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:02,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:02,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:02,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:03,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:03,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:06,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:06,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:06,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:06,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:06,373 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:07,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:07,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:10,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:10,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:10,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:10,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:11,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:11,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:11,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:14,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:14,585 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:14,585 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:14,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:15,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:15,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:15,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:15,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:17,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:18,857 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:18,858 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:18,858 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:18,863 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:19,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:21,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:22,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:22,718 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:22,718 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:22,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:22,723 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:23,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:23,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:25,465 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:28:27,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:27,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:27,143 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:27,147 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:27,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:28,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:29,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:30,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:30,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:30,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:30,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:30,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:30,879 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:30,884 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:31,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:33,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:33,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:35,210 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:35,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:35,211 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:35,216 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:35,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:37,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:39,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:39,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:39,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:39,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:39,202 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:39,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:41,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:43,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:43,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:43,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:43,846 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:44,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:44,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:45,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:45,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:47,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:47,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:47,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:47,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:48,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:49,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:50,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:54,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:54,480 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:54,481 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:54,486 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:55,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:55,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:55,470 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:28:55,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:59,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:59,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:59,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:59,693 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:00,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:00,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:00,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:00,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:01,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:01,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:03,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:03,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:03,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:03,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:04,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:05,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:06,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:07,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:07,644 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:07,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:07,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:08,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:09,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:11,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:12,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:12,296 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:12,296 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:12,301 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:12,321 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:13,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:15,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:15,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:15,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:16,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:16,119 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:16,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:16,124 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:16,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:17,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:17,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:22,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:24,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:24,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:24,702 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:24,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:25,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:25,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:25,469 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:29:27,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:30,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:30,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:30,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:30,811 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:30,812 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:30,812 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:30,817 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:31,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:32,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:33,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:35,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:35,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:35,965 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:35,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:36,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:37,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:37,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:42,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:43,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:43,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:43,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:43,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:44,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:45,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:45,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:47,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:47,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:47,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:47,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:48,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:48,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:49,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:51,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:51,706 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:51,707 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:51,712 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:52,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:53,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:53,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:55,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:29:59,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:00,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:00,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:00,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:00,452 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:00,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:00,453 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:00,458 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:01,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:01,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:04,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:05,707 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:05,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:05,708 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:05,714 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:06,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:07,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:09,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:11,986 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:11,987 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:11,987 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:11,993 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:12,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:13,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:14,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:15,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:15,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:15,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:15,574 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:15,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:15,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:16,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:17,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:19,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:19,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:19,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:19,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:20,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:20,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:21,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:24,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:24,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:24,408 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:24,413 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:24,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:25,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:25,475 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:30:26,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:29,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:29,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:29,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:29,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:29,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:30,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:30,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:30,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:31,438 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:32,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:34,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:34,122 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:34,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:34,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:34,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:35,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:38,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:38,073 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:38,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:38,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:38,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:38,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:39,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:43,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:43,004 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:43,004 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:43,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:43,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:44,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:45,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:45,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:45,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:49,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:49,265 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:49,266 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:49,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:49,271 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:49,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:51,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:53,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:53,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:53,373 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:53,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:53,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:54,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:55,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:55,478 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:30:57,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:57,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:57,830 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:57,836 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:58,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:59,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:59,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:00,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:00,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:00,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:01,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:01,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:01,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:01,788 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:02,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:03,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:05,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:05,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:05,945 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:05,945 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:05,951 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:06,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:07,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:10,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:12,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:12,274 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:12,274 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:12,281 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:12,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:13,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:15,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:16,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:16,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:16,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:16,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:16,699 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:17,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:17,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:21,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:21,532 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:21,532 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:21,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:21,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:22,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:23,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:25,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:31:26,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:26,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:26,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:26,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:26,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:27,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:27,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:30,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:30,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:30,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:32,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:32,131 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:32,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:32,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:32,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:33,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:33,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:35,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:35,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:35,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:35,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:36,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:37,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:38,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:41,819 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:41,820 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:41,821 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:41,826 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:42,539 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:43,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:43,827 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:45,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:45,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:45,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:47,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:47,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:47,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:47,121 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:47,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:49,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:49,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:51,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:51,459 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:51,459 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:51,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:51,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:53,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:54,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:55,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:55,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:55,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:55,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:55,483 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:31:55,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:57,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:59,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:00,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:00,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:00,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:04,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:04,058 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:04,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:04,064 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:04,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:05,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:05,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:09,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:09,050 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:09,050 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:09,056 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:09,580 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:11,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:11,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:12,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:12,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:12,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:12,761 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:13,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:13,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:15,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:16,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:18,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:18,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:18,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:18,158 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:18,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:19,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:22,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:23,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:23,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:23,727 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:23,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:24,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:25,487 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:32:25,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:27,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:27,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:27,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:27,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:27,313 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:27,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:29,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:30,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:30,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:30,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:32,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:32,062 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:32,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:32,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:32,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:33,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:33,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:36,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:36,662 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:36,662 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:36,667 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:37,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:37,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:38,668 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:43,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:44,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:44,966 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:44,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:44,972 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:45,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:45,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:45,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:45,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:45,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:49,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:49,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:49,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:49,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:49,879 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:50,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:51,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:53,647 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:53,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:53,648 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:53,653 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:54,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:54,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:55,490 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:32:55,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:58,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:58,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:58,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:58,519 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:58,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:59,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:00,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:00,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:03,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:03,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:03,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:03,744 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:04,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:05,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:05,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:08,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:08,073 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:08,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:08,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:08,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:09,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:11,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:11,260 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:11,261 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:11,261 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:11,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:11,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:13,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:15,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:15,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:15,850 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:15,851 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:15,851 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:15,857 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:16,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:16,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:17,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:21,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:22,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:22,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:22,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:22,069 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:22,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:23,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:25,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:33:26,806 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:26,807 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:26,807 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:26,813 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:27,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:27,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:27,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:30,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:30,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:30,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:30,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:30,610 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:30,610 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:30,616 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:30,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:31,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:33,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:36,260 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:36,261 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:36,262 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:36,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:36,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:37,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:39,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:41,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:41,269 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:41,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:41,274 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:41,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:43,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:44,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:44,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:45,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:45,000 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:45,006 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:45,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:45,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:45,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:45,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:45,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:49,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:50,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:50,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:50,745 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:50,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:51,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:51,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:54,746 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:54,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:54,747 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:54,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:54,752 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:55,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:33:55,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:55,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:00,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:00,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:03,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:03,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:03,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:03,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:03,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:05,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:05,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:07,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:07,612 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:07,613 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:07,618 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:07,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:09,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:10,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:12,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:12,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:12,667 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:12,672 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:12,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:13,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:15,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:15,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:15,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:16,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:16,490 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:16,491 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:16,491 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:16,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:16,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:17,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:21,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:21,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:21,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:21,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:21,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:22,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:23,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:25,502 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:34:26,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:26,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:26,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:26,330 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:26,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:27,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:27,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:30,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:30,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:30,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:33,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:33,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:33,669 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:33,670 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:33,676 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:33,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:35,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:37,755 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:37,756 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:37,756 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:37,762 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:37,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:38,762 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:39,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:41,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:41,683 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:41,683 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:41,688 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:41,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:43,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:44,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:45,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:45,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:45,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:47,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:47,398 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:47,398 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:47,404 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:47,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:49,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:50,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:51,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:51,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:51,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:51,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:51,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:53,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:55,502 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:34:55,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:59,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:59,768 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:59,768 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:59,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:59,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:00,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:01,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:01,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:05,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:05,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:05,899 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:05,905 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:06,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:06,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:07,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:10,910 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:10,911 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:10,911 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:10,917 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:11,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:11,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:12,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:15,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:15,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:15,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:15,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:15,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:15,617 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:15,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:15,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:17,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:18,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:19,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:19,948 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:19,948 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:19,954 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:20,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:21,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:23,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:24,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:24,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:24,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:24,194 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:24,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:25,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:35:25,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:29,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:29,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:29,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:29,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:29,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:29,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:30,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:31,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:32,792 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:32,792 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:32,792 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:32,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:32,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:33,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:34,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:36,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:36,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:36,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:36,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:36,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:37,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:40,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:42,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:42,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:42,170 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:42,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:42,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:43,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:45,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:45,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:45,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:46,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:48,178 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:48,179 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:48,179 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:48,185 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:48,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:49,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:52,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:52,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:52,557 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:52,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:52,563 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:52,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:53,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:55,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:35:57,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:57,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:57,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:57,795 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:57,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:57,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:59,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:00,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:00,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:02,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:02,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:02,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:02,338 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:02,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:03,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:03,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:06,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:06,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:06,745 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:06,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:06,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:07,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:08,752 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:11,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:11,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:11,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:11,279 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:11,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:13,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:14,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:15,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:15,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:15,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:15,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:15,454 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:15,454 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:15,460 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:15,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:17,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:19,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:19,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:19,851 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:19,852 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:19,857 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:19,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:21,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:23,842 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:23,843 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:23,843 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:23,848 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:23,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:24,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:25,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:36:25,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:30,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:30,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:30,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:30,875 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:30,880 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:30,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:31,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:34,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:34,601 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:34,601 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:34,607 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:34,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:35,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:35,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:38,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:38,551 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:38,551 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:38,556 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:38,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:39,982 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:41,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:42,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:42,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:42,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:42,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:42,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:43,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:45,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:45,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:45,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:46,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:46,186 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:46,186 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:46,192 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:46,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:47,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:47,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:50,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:50,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:50,144 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:50,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:50,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:51,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:53,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:53,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:53,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:53,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:53,947 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:54,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:55,513 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:36:56,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:57,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:57,182 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:57,182 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:57,188 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:58,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:58,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:58,188 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:00,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:00,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:00,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:00,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:00,742 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:00,743 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:00,748 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:01,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:02,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:03,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:06,156 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:06,157 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:06,157 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:06,163 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:07,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:08,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:09,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:11,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:11,902 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:11,902 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:11,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:12,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:14,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:14,908 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:15,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:15,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:15,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:16,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:16,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:16,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:16,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:17,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:18,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:20,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:21,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:21,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:21,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:21,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:22,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:22,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:25,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:37:25,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:25,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:25,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:25,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:25,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:26,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:28,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:29,679 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:29,680 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:29,680 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:29,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:30,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:30,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:30,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:31,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:32,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:33,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:33,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:33,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:33,191 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:34,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:34,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:37,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:38,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:38,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:38,170 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:38,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:39,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:40,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:42,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:42,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:42,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:42,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:42,559 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:43,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:44,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:45,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:45,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:45,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:46,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:46,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:46,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:46,950 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:47,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:47,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:48,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:51,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:51,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:51,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:51,750 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:52,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:53,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:54,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:55,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:37:55,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:55,570 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:55,571 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:55,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:56,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:58,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:59,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:00,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:00,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:00,035 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:00,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:00,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:00,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:00,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:00,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:02,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:04,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:04,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:04,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:04,808 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:04,814 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:05,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:06,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:09,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:09,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:09,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:09,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:10,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:10,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:12,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:13,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:13,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:13,371 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:13,377 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:14,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:14,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:15,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:15,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:15,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:16,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:19,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:19,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:19,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:19,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:20,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:21,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:22,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:23,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:23,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:23,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:23,515 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:24,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:25,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:38:26,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:27,523 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:28,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:28,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:28,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:28,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:29,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:30,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:30,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:30,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:31,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:31,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:31,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:31,958 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:32,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:32,997 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:34,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:35,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:35,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:35,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:35,658 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:36,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:38,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:38,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:39,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:39,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:39,824 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:39,830 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:40,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:42,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:43,831 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:45,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:45,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:45,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:45,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:45,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:45,865 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:45,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:46,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:48,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:48,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:49,995 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:49,996 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:49,996 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:50,002 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:50,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:52,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:54,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:54,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:54,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:54,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:54,059 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:54,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:55,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:38:56,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:58,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:58,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:58,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:58,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:58,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:59,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:00,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:00,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:00,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:00,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:01,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:01,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:01,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:01,812 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:02,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:04,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:04,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:05,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:05,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:05,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:05,990 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:06,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:08,201 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:09,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:09,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:09,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:09,991 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:09,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:10,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:12,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:14,998 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:15,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:15,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:15,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:18,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:18,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:18,385 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:18,390 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:19,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:20,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:20,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:24,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:24,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:24,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:24,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:25,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:25,527 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:39:25,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:26,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:29,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:29,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:29,572 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:29,578 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:30,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:30,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:30,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:30,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:31,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:32,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:33,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:33,422 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:33,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:33,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:34,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:34,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:36,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:38,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:38,112 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:38,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:38,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:38,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:40,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:42,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:42,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:42,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:42,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:42,059 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:42,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:44,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:45,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:45,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:45,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:46,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:46,040 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:46,040 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:46,046 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:46,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:47,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:48,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:50,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:50,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:50,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:50,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:50,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:52,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:53,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:54,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:54,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:54,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:54,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:55,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:55,532 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:39:56,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:58,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:59,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:59,248 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:59,248 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:59,254 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:59,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:00,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:00,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:00,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:00,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:03,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:03,859 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:03,859 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:03,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:03,864 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:04,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:06,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:07,935 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:07,936 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:07,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:07,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:08,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:08,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:10,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:12,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:12,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:12,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:12,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:12,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:14,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:14,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:15,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:15,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:15,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:15,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:15,636 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:15,636 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:15,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:16,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:18,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:19,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:20,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:20,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:20,719 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:20,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:21,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:22,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:24,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:24,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:24,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:24,552 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:25,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:25,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:40:25,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:26,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:28,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:28,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:28,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:28,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:29,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:30,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:30,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:30,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:30,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:31,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:34,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:34,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:34,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:34,071 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:34,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:36,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:37,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:38,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:38,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:38,764 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:38,769 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:39,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:40,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:42,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:45,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:45,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:45,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:46,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:46,936 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:46,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:46,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:47,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:47,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:48,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:50,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:50,704 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:50,705 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:50,710 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:51,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:52,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:53,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:54,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:54,716 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:54,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:54,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:55,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:55,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:40:56,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:59,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:00,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:00,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:00,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:00,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:00,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:00,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:00,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:01,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:02,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:04,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:09,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:09,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:09,039 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:09,045 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:09,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:10,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:10,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:13,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:13,060 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:13,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:13,066 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:13,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:14,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:15,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:15,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:15,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:15,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:18,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:18,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:18,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:18,717 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:19,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:20,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:20,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:23,920 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:23,921 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:23,921 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:23,927 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:24,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:25,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:41:26,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:26,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:28,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:28,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:28,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:28,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:28,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:30,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:30,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:30,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:30,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:31,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:31,823 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:31,823 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:31,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:31,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:32,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:34,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:36,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:37,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:37,608 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:37,608 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:37,614 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:38,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:38,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:42,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:42,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:42,148 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:42,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:42,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:42,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:44,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:45,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:45,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:45,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:45,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:45,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:45,919 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:45,924 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:46,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:47,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:48,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:52,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:54,538 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:54,539 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:54,539 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:54,545 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:55,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:55,539 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:41:56,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:58,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:58,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:58,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:58,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:58,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:58,447 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:00,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:00,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:00,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:00,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:02,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:02,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:02,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:02,337 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:02,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:03,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:04,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:05,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:05,913 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:05,914 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:05,919 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:06,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:08,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:08,920 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:12,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:12,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:12,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:12,848 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:13,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:14,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:14,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:15,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:15,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:15,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:17,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:17,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:17,778 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:17,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:18,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:20,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:20,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:21,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:21,939 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:21,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:21,944 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:22,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:24,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:25,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:42:25,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:25,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:25,894 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:25,894 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:25,899 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:26,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:28,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:30,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:30,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:30,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:31,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:32,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:32,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:32,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:32,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:33,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:34,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:36,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:38,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:38,067 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:38,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:38,073 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:38,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:40,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:41,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:41,613 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:41,614 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:41,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:42,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:42,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:42,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:45,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:45,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:45,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:46,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:46,807 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:46,807 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:46,813 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:47,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:47,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:48,522 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:51,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:51,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:51,215 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:51,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:51,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:52,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:53,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:55,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:42:55,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:55,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:55,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:55,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:56,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:58,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:58,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:00,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:00,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:00,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:01,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:01,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:01,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:01,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:01,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:02,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:04,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:04,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:04,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:04,882 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:04,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:05,546 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:06,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:09,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:10,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:10,602 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:10,602 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:10,608 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:11,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:12,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:15,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:15,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:15,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:15,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:19,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:19,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:19,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:19,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:19,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:20,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:20,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:23,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:23,350 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:23,350 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:23,356 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:23,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:24,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:25,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:25,549 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:43:27,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:27,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:27,234 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:27,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:27,578 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:28,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:30,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:30,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:30,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:31,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:31,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:31,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:31,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:31,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:31,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:32,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:35,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:35,064 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:35,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:35,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:35,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:36,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:36,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:41,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:43,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:43,643 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:43,643 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:43,648 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:44,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:44,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:45,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:45,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:45,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:46,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:47,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:47,818 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:47,818 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:47,824 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:48,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:50,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:51,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:53,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:53,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:53,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:53,252 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:53,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:54,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:55,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:43:57,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:57,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:57,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:57,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:57,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:57,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:58,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:00,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:00,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:00,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:02,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:03,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:03,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:03,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:03,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:03,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:04,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:07,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:08,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:08,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:08,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:08,312 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:08,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:10,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:13,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:13,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:13,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:13,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:13,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:14,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:15,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:15,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:15,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:16,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:17,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:17,328 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:17,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:17,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:17,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:18,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:18,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:21,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:21,154 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:21,155 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:21,160 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:21,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:22,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:24,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:24,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:24,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:24,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:24,958 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:25,557 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:44:25,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:26,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:28,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:28,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:28,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:28,848 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:29,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:29,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:30,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:30,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:30,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:30,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:32,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:32,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:32,927 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:32,933 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:33,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:34,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:34,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:37,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:37,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:37,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:37,095 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:37,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:38,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:40,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:40,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:40,713 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:40,713 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:40,719 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:41,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:42,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:45,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:45,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:45,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:45,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:45,819 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:45,820 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:45,820 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:45,826 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:46,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:48,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:49,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:49,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:49,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:49,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:49,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:50,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:50,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:55,254 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:55,254 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:55,255 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:55,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:55,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:44:55,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:56,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:56,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:59,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:59,859 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:59,859 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:59,865 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:00,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:00,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:00,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:00,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:02,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:02,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:03,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:03,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:03,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:03,730 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:04,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:04,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:07,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:12,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:12,161 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:12,161 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:12,167 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:12,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:13,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:14,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:15,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:15,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:15,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:16,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:16,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:16,348 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:16,353 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:16,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:18,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:18,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:23,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:24,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:24,742 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:24,743 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:24,750 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:24,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:25,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:45:26,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:28,563 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:29,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:29,019 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:29,020 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:29,025 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:29,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:30,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:30,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:30,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:30,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:33,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:33,244 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:33,244 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:33,250 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:33,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:34,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:34,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:38,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:38,254 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:38,254 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:38,260 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:38,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:40,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:40,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:41,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:41,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:41,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:41,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:42,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:42,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:45,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:45,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:45,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:45,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:45,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:45,787 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:45,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:45,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:46,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:46,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:50,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:50,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:50,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:50,324 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:50,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:51,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:52,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:55,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:55,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:55,414 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:55,419 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:55,560 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:45:55,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:56,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:56,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:59,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:59,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:59,395 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:59,401 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:59,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:00,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:00,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:00,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:02,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:03,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:03,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:03,420 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:03,427 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:03,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:04,809 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:07,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:12,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:13,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:13,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:13,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:13,018 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:13,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:14,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:15,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:15,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:15,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:17,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:17,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:17,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:17,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:17,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:18,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:18,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:23,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:23,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:23,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:23,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:23,190 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:23,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:24,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:25,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:46:27,978 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:27,979 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:27,979 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:27,984 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:28,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:28,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:30,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:30,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:30,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:30,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:34,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:36,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:36,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:36,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:36,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:36,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:38,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:39,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:40,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:40,444 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:40,444 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:40,450 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:40,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:42,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:44,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:45,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:45,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:45,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:46,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:46,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:46,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:46,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:46,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:48,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:50,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:50,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:50,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:50,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:50,338 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:50,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:52,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:54,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:54,972 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:54,972 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:54,978 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:55,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:46:55,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:55,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:56,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:00,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:00,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:00,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:00,882 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:00,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:00,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:00,890 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:00,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:02,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:05,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:05,862 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:05,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:05,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:05,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:06,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:06,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:09,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:09,845 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:09,845 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:09,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:09,902 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:10,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:12,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:14,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:14,603 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:14,603 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:14,609 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:14,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:15,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:15,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:15,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:16,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:18,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:18,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:18,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:18,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:18,338 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:18,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:20,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:23,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:23,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:23,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:23,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:23,651 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:23,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:24,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:25,569 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:47:28,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:29,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:29,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:29,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:29,333 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:29,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:30,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:30,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:30,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:30,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:33,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:33,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:33,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:33,087 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:33,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:34,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:34,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:37,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:37,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:37,031 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:37,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:37,942 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:38,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:40,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:41,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:41,529 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:41,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:41,535 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:41,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:42,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:45,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:45,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:45,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:45,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:47,042 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:47,042 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:47,042 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:47,048 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:47,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:48,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:51,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:53,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:53,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:53,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:53,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:53,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:54,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:55,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:47:56,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:00,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:01,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:01,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:01,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:01,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:01,646 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:01,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:02,982 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:05,611 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:05,612 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:05,612 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:05,618 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:05,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:06,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:07,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:10,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:10,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:10,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:10,887 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:10,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:12,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:12,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:15,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:15,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:15,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:15,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:15,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:15,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:15,338 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:15,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:17,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:18,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:18,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:18,914 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:18,914 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:18,919 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:19,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:21,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:22,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:22,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:22,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:22,359 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:23,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:23,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:25,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:25,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:48:26,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:26,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:26,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:26,326 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:27,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:29,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:29,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:30,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:30,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:30,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:31,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:31,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:31,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:31,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:32,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:33,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:35,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:35,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:35,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:35,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:35,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:36,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:37,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:40,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:44,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:44,163 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:44,164 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:44,169 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:45,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:45,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:45,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:45,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:46,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:47,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:48,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:48,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:48,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:48,374 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:49,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:51,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:51,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:51,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:51,892 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:51,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:51,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:52,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:53,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:55,582 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:48:56,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:58,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:58,008 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:58,009 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:58,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:58,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:59,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:00,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:02,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:06,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:06,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:06,239 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:06,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:07,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:08,281 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:09,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:11,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:11,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:11,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:11,773 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:12,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:13,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:13,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:15,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:15,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:15,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:15,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:15,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:15,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:15,392 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:16,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:17,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:19,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:19,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:19,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:19,699 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:19,704 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:20,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:21,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:23,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:23,900 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:23,900 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:23,906 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:24,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:24,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:25,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:25,582 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:49:27,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:27,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:27,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:27,669 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:28,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:29,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:30,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:30,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:30,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:30,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:32,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:32,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:32,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:32,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:32,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:33,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:36,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:40,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:40,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:40,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:40,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:41,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:41,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:43,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:44,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:44,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:44,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:44,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:45,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:45,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:45,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:45,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:46,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:47,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:48,675 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:48,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:48,676 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:48,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:49,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:51,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:51,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:52,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:52,519 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:52,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:52,525 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:53,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:55,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:55,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:49:57,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:58,966 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:58,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:58,967 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:58,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:59,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:00,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:00,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:01,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:03,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:03,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:03,625 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:03,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:03,631 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:04,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:05,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:07,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:07,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:07,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:07,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:08,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:08,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:09,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:11,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:11,861 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:11,861 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:11,866 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:12,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:13,168 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:13,867 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:15,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:15,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:15,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:16,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:16,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:16,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:16,253 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:17,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:19,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:19,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:21,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:21,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:21,034 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:21,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:21,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:23,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:25,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:25,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:50:26,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:26,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:26,566 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:26,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:27,187 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:29,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:30,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:30,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:30,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:30,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:30,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:30,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:30,517 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:30,522 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:31,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:33,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:34,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:34,534 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:34,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:34,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:35,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:35,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:37,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:39,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:39,071 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:39,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:39,076 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:39,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:41,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:41,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:43,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:43,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:43,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:43,092 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:43,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:45,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:45,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:45,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:45,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:46,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:48,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:48,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:48,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:48,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:48,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:49,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:52,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:52,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:52,345 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:52,346 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:52,351 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:53,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:55,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:55,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:50:56,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:56,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:56,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:56,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:57,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:57,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:59,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:00,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:00,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:00,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:02,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:02,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:02,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:02,102 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:02,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:03,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:03,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:07,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:07,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:07,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:07,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:08,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:08,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:09,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:11,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:11,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:11,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:11,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:12,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:13,253 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:14,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:15,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:15,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:15,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:15,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:15,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:15,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:15,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:16,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:17,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:19,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:22,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:22,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:22,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:22,780 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:23,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:25,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:25,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:51:25,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:27,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:27,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:27,544 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:27,550 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:28,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:29,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:30,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:30,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:30,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:31,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:31,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:31,699 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:31,699 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:31,705 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:32,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:33,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:36,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:36,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:36,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:36,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:36,341 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:37,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:39,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:41,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:41,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:41,814 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:41,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:41,820 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:42,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:45,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:45,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:45,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:45,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:46,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:46,159 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:46,159 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:46,166 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:46,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:47,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:49,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:50,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:50,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:50,206 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:50,211 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:50,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:52,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:53,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:55,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:55,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:55,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:55,439 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:55,596 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:51:56,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:57,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:57,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:59,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:59,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:59,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:59,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:59,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:00,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:00,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:00,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:01,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:03,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:03,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:03,642 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:03,642 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:03,648 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:04,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:05,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:08,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:08,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:08,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:08,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:08,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:09,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:11,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:11,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:11,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:11,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:11,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:12,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:12,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:13,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:15,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:15,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:15,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:15,736 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:15,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:15,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:15,742 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:16,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:16,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:18,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:20,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:20,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:20,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:20,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:20,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:22,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:23,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:23,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:23,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:23,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:23,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:24,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:25,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:52:26,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:27,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:27,656 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:27,656 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:27,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:28,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:28,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:29,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:30,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:30,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:30,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:34,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:34,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:34,526 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:34,532 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:35,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:35,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:36,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:38,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:38,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:38,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:38,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:39,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:40,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:40,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:43,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:43,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:43,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:43,121 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:43,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:44,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:45,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:45,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:45,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:46,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:47,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:47,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:47,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:47,644 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:48,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:48,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:51,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:52,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:52,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:52,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:52,466 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:53,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:54,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:55,602 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:52:57,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:00,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:00,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:00,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:01,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:01,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:01,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:01,130 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:01,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:02,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:03,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:04,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:04,562 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:04,562 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:04,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:05,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:06,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:08,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:11,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:11,258 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:11,259 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:11,264 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:11,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:12,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:14,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:15,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:15,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:15,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:16,808 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:16,812 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:16,812 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:16,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:17,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:18,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:19,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:24,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:24,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:24,210 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:24,216 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:24,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:25,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:25,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:53:26,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:27,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:27,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:27,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:27,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:28,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:28,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:30,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:30,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:30,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:30,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:31,178 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:31,178 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:31,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:31,184 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:31,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:32,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:35,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:35,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:35,656 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:35,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:35,661 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:36,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:36,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:39,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:39,495 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:39,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:39,500 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:40,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:40,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:41,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:44,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:44,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:44,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:44,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:44,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:45,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:45,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:45,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:46,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:47,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:52,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:52,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:52,406 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:52,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:52,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:52,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:54,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:55,609 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:53:56,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:56,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:56,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:56,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:57,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:57,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:58,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:00,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:00,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:00,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:01,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:01,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:01,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:01,157 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:01,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:02,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:03,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:05,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:05,559 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:05,559 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:05,565 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:06,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:06,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:08,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:10,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:10,187 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:10,187 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:10,192 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:10,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:12,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:14,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:15,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:15,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:15,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:15,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:15,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:15,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:15,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:16,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:16,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:19,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:19,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:19,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:19,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:19,377 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:19,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:20,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:23,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:23,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:23,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:23,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:23,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:24,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:24,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:25,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:54:28,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:28,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:28,622 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:28,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:29,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:29,628 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:30,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:30,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:30,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:30,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:35,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:35,970 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:35,971 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:35,971 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:35,976 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:36,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:36,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:39,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:39,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:39,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:39,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:40,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:40,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:40,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:43,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:43,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:43,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:43,930 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:44,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:44,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:45,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:45,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:45,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:46,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:50,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:50,470 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:50,471 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:50,476 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:50,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:51,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:52,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:55,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:55,059 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:55,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:55,066 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:55,580 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:55,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:54:56,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:56,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:59,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:59,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:59,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:59,880 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:00,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:00,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:00,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:00,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:00,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:02,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:03,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:03,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:03,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:03,838 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:04,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:04,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:07,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:12,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:12,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:12,239 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:12,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:12,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:13,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:14,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:15,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:15,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:15,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:16,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:16,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:16,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:16,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:17,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:18,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:18,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:20,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:20,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:20,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:20,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:21,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:22,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:24,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:24,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:24,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:24,885 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:24,891 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:25,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:55:25,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:26,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:29,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:29,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:29,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:29,052 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:29,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:30,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:30,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:30,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:30,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:30,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:32,752 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:32,752 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:32,753 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:32,758 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:33,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:34,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:35,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:36,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:36,557 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:36,558 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:36,563 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:36,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:38,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:41,423 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:41,424 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:41,425 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:41,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:41,430 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:41,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:42,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:45,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:45,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:45,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:45,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:45,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:45,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:45,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:45,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:46,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:46,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:49,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:49,748 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:49,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:49,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:50,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:50,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:51,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:54,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:54,530 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:54,530 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:54,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:54,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:55,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:55:56,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:57,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:58,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:58,075 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:58,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:58,080 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:58,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:58,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:00,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:00,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:00,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:03,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:04,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:04,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:04,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:04,006 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:04,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:04,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:09,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:12,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:12,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:12,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:12,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:12,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:14,231 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:14,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:15,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:15,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:15,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:16,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:16,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:16,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:16,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:16,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:18,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:19,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:21,539 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:21,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:21,540 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:21,546 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:21,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:22,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:24,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:25,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:25,501 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:25,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:25,507 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:25,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:56:25,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:26,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:29,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:30,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:30,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:30,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:30,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:30,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:30,962 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:30,968 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:31,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:32,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:35,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:35,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:35,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:35,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:35,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:35,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:36,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:40,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:41,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:41,002 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:41,003 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:41,008 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:41,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:42,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:45,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:45,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:45,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:45,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:45,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:45,712 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:45,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:45,718 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:45,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:46,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:50,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:50,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:50,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:50,798 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:50,803 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:51,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:52,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:55,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:56:56,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:57,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:57,639 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:57,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:57,645 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:57,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:58,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:00,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:00,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:00,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:01,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:01,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:01,726 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:01,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:01,732 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:01,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:02,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:05,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:05,596 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:05,596 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:05,602 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:05,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:06,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:07,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:09,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:09,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:09,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:09,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:09,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:10,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:12,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:14,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:14,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:14,370 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:14,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:14,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:15,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:15,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:15,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:16,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:18,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:18,132 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:18,132 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:18,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:18,138 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:18,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:18,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:22,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:22,336 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:22,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:22,342 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:22,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:23,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:24,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:25,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:57:28,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:28,130 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:28,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:28,136 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:28,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:28,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:29,136 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:30,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:30,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:30,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:32,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:32,010 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:32,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:32,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:32,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:32,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:35,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:36,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:36,203 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:36,203 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:36,209 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:36,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:36,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:40,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:40,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:40,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:40,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:40,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:40,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:40,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:43,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:43,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:43,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:43,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:43,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:44,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:45,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:45,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:45,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:45,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:50,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:51,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:51,295 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:51,295 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:51,300 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:51,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:52,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:55,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:55,630 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:57:56,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:56,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:56,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:56,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:56,849 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:58,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:00,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:00,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:00,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:00,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:05,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:05,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:05,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:05,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:05,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:06,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:06,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:09,707 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:09,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:09,708 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:09,714 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:09,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:10,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:11,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:13,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:13,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:13,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:13,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:13,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:14,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:15,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:15,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:15,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:17,274 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:17,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:17,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:17,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:17,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:17,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:18,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:21,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:21,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:21,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:21,959 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:22,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:22,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:22,960 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:25,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:58:28,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:28,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:28,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:28,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:28,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:28,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:28,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:30,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:30,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:30,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:32,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:32,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:32,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:32,222 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:32,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:32,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:33,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:36,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:36,265 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:36,265 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:36,271 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:36,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:36,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:38,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:39,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:39,866 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:39,866 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:39,872 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:39,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:40,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:43,872 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:45,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:45,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:45,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:48,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:48,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:48,375 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:48,381 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:48,927 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:49,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:50,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:52,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:52,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:52,865 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:52,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:52,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:54,871 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:54,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:55,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:58:56,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:56,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:56,626 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:56,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:56,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:58,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:00,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:00,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:00,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:00,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:01,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:01,340 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:01,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:01,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:01,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:02,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:05,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:07,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:07,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:07,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:07,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:07,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:08,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:11,156 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:11,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:11,367 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:11,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:11,372 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:11,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:12,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:15,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:15,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:15,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:15,365 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:15,365 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:15,370 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:15,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:16,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:16,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:18,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:18,913 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:18,913 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:18,918 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:18,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:20,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:21,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:24,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:24,644 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:24,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:24,649 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:24,979 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:25,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:59:26,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:27,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:28,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:28,290 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:28,290 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:28,295 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:28,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:28,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:30,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:30,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:30,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:32,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:32,093 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:32,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:32,098 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:32,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:32,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:33,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:38,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:39,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:39,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:39,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:39,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:40,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:41,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:43,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:43,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:43,851 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:43,852 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:43,857 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:44,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:45,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:45,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:45,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:48,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:48,429 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:48,429 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:48,434 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:49,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:49,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:51,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:54,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:54,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:54,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:54,774 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:54,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:55,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:55,648 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:59:57,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:58,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:58,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:58,721 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:58,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:59,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:59,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:00,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:00,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:00,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:01,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:05,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:07,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:07,202 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:07,202 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:07,208 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:08,043 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:09,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:11,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:11,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:11,342 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:11,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:11,347 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:12,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:13,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:15,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:15,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:15,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:16,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:16,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:16,367 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:16,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:16,372 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:17,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:19,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:20,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:20,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:20,684 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:20,689 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:21,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:21,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:23,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:24,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:24,529 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:24,530 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:24,534 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:25,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:25,644 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:00:27,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:27,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:30,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:30,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:30,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:30,006 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:30,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:30,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:30,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:30,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:31,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:33,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:33,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:33,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:33,927 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:33,932 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:34,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:35,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:37,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:37,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:37,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:37,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:38,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:38,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:39,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:41,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:41,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:41,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:41,846 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:42,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:43,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:43,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:45,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:46,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:46,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:46,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:46,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:47,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:49,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:49,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:51,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:51,921 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:51,922 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:51,926 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:52,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:53,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:54,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:55,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:00:56,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:56,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:56,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:56,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:56,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:57,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:00,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:00,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:00,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:00,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:01,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:01,157 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:01,157 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:01,163 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:02,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:03,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:05,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:05,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:05,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:05,313 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:06,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:06,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:07,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:11,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:13,408 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:13,409 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:13,409 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:13,414 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:14,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:15,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:15,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:15,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:15,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:16,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:17,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:17,319 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:17,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:17,324 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:18,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:19,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:21,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:21,277 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:21,278 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:21,283 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:22,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:22,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:23,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:25,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:01:25,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:25,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:25,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:25,978 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:26,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:27,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:27,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:29,670 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:29,670 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:29,670 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:29,676 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:30,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:30,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:30,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:30,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:31,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:33,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:37,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:37,918 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:37,918 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:37,923 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:38,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:38,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:39,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:42,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:42,491 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:42,491 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:42,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:43,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:44,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:45,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:45,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:46,900 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:46,901 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:46,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:46,907 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:47,189 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:49,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:49,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:51,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:51,059 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:51,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:51,064 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:51,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:53,199 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:55,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:55,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:55,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:55,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:55,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:55,656 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:01:56,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:57,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:00,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:00,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:00,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:00,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:01,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:01,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:01,207 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:01,212 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:02,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:03,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:06,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:07,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:07,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:07,215 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:07,220 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:08,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:09,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:11,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:11,297 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:11,297 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:11,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:11,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:12,225 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:13,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:15,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:15,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:15,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:15,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:15,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:15,434 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:15,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:16,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:16,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:17,232 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:21,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:21,422 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:21,422 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:21,427 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:22,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:22,427 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:23,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:25,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:25,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:25,651 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:25,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:25,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:02:26,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:27,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:27,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:30,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:30,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:30,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:30,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:30,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:30,745 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:30,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:31,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:32,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:33,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:35,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:35,855 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:35,855 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:35,860 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:36,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:37,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:37,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:41,652 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:41,653 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:41,654 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:41,659 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:42,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:43,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:43,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:45,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:45,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:45,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:45,945 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:45,945 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:45,950 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:46,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:47,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:48,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:49,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:49,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:49,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:49,736 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:50,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:51,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:53,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:53,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:53,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:53,557 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:54,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:54,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:55,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:55,658 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:02:57,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:57,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:57,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:57,625 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:58,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:59,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:59,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:00,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:00,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:00,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:01,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:01,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:01,492 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:01,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:02,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:03,299 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:05,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:06,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:06,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:06,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:06,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:07,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:07,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:10,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:10,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:10,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:10,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:10,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:11,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:13,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:14,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:14,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:14,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:14,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:15,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:15,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:15,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:15,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:16,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:17,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:19,736 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:19,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:19,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:19,742 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:20,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:21,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:21,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:23,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:23,990 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:23,991 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:23,996 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:24,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:25,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:25,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:03:27,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:28,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:28,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:28,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:28,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:28,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:29,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:30,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:30,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:30,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:32,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:32,200 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:32,200 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:32,206 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:32,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:33,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:33,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:37,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:37,033 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:37,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:37,038 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:37,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:39,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:39,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:44,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:45,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:45,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:45,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:45,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:45,400 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:45,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:45,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:46,361 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:47,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:48,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:48,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:48,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:48,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:49,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:49,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:51,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:52,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:52,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:52,796 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:52,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:53,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:55,375 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:55,665 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:03:55,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:57,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:57,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:57,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:57,252 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:57,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:59,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:00,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:00,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:00,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:01,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:01,472 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:01,473 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:01,473 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:01,478 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:02,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:03,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:06,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:06,076 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:06,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:06,081 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:06,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:07,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:07,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:12,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:12,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:12,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:12,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:12,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:12,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:13,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:15,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:15,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:15,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:16,867 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:16,871 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:16,871 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:16,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:17,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:17,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:19,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:22,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:23,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:23,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:23,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:23,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:24,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:25,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:25,669 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:04:27,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:27,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:27,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:27,762 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:28,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:28,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:29,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:30,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:30,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:30,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:32,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:32,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:32,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:32,282 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:32,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:33,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:34,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:36,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:36,159 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:36,160 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:36,165 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:36,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:37,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:40,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:40,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:40,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:40,304 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:40,309 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:40,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:41,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:44,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:44,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:44,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:44,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:45,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:45,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:45,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:45,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:45,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:47,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:48,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:48,673 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:48,673 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:48,678 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:49,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:50,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:51,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:52,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:52,474 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:52,474 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:52,479 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:53,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:53,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:55,672 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:04:56,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:58,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:58,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:58,404 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:58,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:58,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:59,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:00,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:00,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:00,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:02,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:02,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:02,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:02,805 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:02,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:03,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:05,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:07,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:07,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:07,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:07,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:07,489 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:08,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:09,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:11,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:11,154 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:11,154 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:11,159 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:11,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:13,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:14,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:15,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:15,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:15,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:16,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:16,889 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:16,889 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:16,894 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:17,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:19,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:19,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:22,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:22,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:22,214 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:22,222 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:22,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:23,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:25,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:25,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:05:26,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:26,340 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:26,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:26,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:26,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:27,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:30,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:30,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:30,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:30,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:34,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:34,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:34,647 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:34,652 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:35,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:35,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:35,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:38,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:38,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:38,696 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:38,701 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:39,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:39,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:40,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:42,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:42,538 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:42,538 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:42,544 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:43,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:43,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:45,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:45,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:45,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:46,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:49,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:49,097 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:49,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:49,103 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:49,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:51,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:52,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:54,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:54,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:54,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:54,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:54,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:55,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:55,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:05:57,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:59,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:59,892 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:59,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:59,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:00,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:00,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:00,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:00,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:01,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:03,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:04,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:04,468 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:04,468 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:04,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:04,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:05,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:08,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:08,333 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:08,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:08,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:08,339 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:08,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:09,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:12,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:12,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:12,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:12,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:12,578 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:13,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:14,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:15,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:15,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:15,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:17,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:17,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:17,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:17,126 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:17,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:19,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:19,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:23,232 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:23,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:23,233 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:23,238 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:23,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:24,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:25,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:25,683 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:06:27,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:27,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:27,556 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:27,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:27,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:29,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:29,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:30,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:30,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:30,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:31,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:31,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:31,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:31,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:31,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:33,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:35,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:35,164 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:35,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:35,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:35,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:35,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:37,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:39,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:39,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:39,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:39,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:39,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:40,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:41,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:43,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:43,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:43,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:43,157 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:43,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:45,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:45,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:45,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:45,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:45,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:47,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:47,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:47,192 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:47,197 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:47,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:49,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:51,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:51,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:51,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:51,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:51,195 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:51,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:53,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:54,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:54,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:54,772 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:54,777 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:55,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:55,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:55,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:06:56,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:59,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:59,161 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:59,162 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:59,166 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:59,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:00,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:00,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:00,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:01,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:02,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:03,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:03,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:03,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:03,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:03,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:05,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:07,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:07,815 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:07,815 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:07,815 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:07,820 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:08,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:09,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:12,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:12,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:12,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:12,242 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:12,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:13,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:13,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:15,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:15,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:15,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:15,824 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:15,825 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:15,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:15,830 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:16,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:17,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:18,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:19,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:19,894 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:19,894 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:19,899 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:20,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:21,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:23,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:23,257 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:23,257 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:23,262 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:23,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:24,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:25,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:25,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:07:29,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:30,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:30,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:30,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:30,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:30,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:30,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:30,744 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:31,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:31,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:34,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:34,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:34,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:34,373 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:34,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:35,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:35,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:38,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:38,221 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:38,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:38,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:38,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:39,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:41,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:42,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:42,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:42,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:42,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:42,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:43,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:45,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:45,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:45,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:45,911 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:45,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:45,912 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:45,917 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:46,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:46,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:47,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:50,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:50,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:50,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:50,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:51,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:51,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:52,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:54,735 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:54,735 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:54,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:54,740 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:55,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:07:55,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:55,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:58,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:59,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:59,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:59,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:59,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:59,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:00,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:00,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:00,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:01,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:02,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:02,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:02,870 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:02,875 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:03,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:03,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:03,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:06,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:06,846 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:06,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:06,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:07,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:07,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:09,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:10,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:10,610 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:10,610 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:10,615 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:10,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:11,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:15,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:15,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:15,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:15,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:20,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:24,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:24,441 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:24,441 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:24,446 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:24,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:25,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:25,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:08:25,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:28,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:28,442 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:28,442 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:28,447 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:28,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:29,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:30,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:30,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:30,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:31,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:32,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:32,381 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:32,381 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:32,386 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:32,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:33,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:36,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:36,539 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:36,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:36,540 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:36,545 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:36,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:37,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:41,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:41,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:41,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:41,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:41,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:41,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:43,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:45,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:45,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:45,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:45,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:45,849 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:45,849 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:45,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:46,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:46,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:47,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:49,708 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:49,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:49,709 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:49,714 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:49,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:51,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:52,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:53,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:53,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:53,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:53,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:53,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:55,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:08:55,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:58,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:59,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:59,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:59,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:59,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:59,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:00,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:00,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:00,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:01,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:03,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:03,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:03,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:03,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:03,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:04,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:05,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:08,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:08,336 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:08,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:08,341 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:08,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:09,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:10,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:15,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:15,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:15,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:15,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:16,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:16,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:16,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:16,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:16,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:17,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:20,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:21,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:21,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:21,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:21,342 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:21,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:23,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:25,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:09:26,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:26,608 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:26,609 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:26,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:26,613 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:26,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:27,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:30,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:30,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:30,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:30,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:30,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:30,599 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:30,604 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:30,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:31,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:31,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:36,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:37,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:37,550 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:37,550 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:37,555 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:37,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:39,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:42,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:42,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:42,385 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:42,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:42,390 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:42,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:43,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:45,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:45,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:45,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:47,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:49,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:49,097 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:49,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:49,102 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:49,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:51,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:53,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:53,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:53,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:53,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:53,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:53,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:55,699 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:09:55,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:58,700 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:00,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:00,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:00,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:01,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:01,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:01,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:01,440 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:01,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:03,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:04,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:06,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:06,132 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:06,133 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:06,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:06,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:07,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:10,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:10,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:10,278 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:10,279 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:10,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:10,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:11,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:14,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:14,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:14,813 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:14,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:14,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:15,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:15,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:15,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:15,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:15,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:19,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:19,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:19,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:19,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:19,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:20,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:21,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:24,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:24,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:24,508 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:24,513 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:24,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:25,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:10:25,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:26,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:28,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:28,093 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:28,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:28,098 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:28,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:29,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:30,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:30,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:30,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:32,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:32,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:32,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:32,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:32,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:32,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:33,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:37,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:38,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:38,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:38,256 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:38,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:38,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:39,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:43,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:44,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:44,415 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:44,415 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:44,420 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:44,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:45,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:45,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:45,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:45,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:48,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:51,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:51,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:51,350 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:51,354 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:51,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:53,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:53,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:55,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:10:57,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:57,272 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:57,272 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:57,277 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:57,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:59,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:59,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:00,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:00,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:00,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:01,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:01,203 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:01,203 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:01,208 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:01,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:03,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:05,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:05,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:05,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:05,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:05,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:05,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:05,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:09,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:09,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:09,029 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:09,034 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:09,982 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:09,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:11,035 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:14,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:14,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:14,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:14,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:14,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:15,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:15,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:15,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:15,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:16,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:19,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:19,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:19,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:19,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:19,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:19,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:22,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:25,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:25,581 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:25,581 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:25,586 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:25,707 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:11:25,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:27,708 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:28,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:30,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:30,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:30,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:30,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:30,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:30,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:30,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:31,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:32,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:33,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:35,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:35,710 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:35,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:35,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:36,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:38,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:38,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:39,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:39,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:39,518 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:39,523 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:40,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:42,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:44,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:45,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:45,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:45,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:45,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:45,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:45,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:45,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:46,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:48,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:49,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:50,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:50,513 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:50,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:50,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:51,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:52,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:54,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:54,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:54,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:54,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:55,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:55,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:55,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:11:56,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:58,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:58,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:58,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:58,799 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:59,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:00,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:00,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:00,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:00,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:01,284 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:02,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:02,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:02,405 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:02,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:03,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:04,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:06,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:09,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:09,330 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:09,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:09,335 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:10,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:12,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:12,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:12,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:12,884 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:12,884 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:12,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:13,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:14,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:15,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:15,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:15,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:16,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:16,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:16,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:16,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:17,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:17,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:18,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:21,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:21,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:21,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:21,390 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:22,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:23,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:24,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:25,713 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:12:26,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:26,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:26,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:26,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:27,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:28,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:28,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:30,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:30,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:30,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:34,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:35,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:35,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:35,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:35,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:36,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:38,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:39,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:39,010 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:39,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:39,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:39,075 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:40,016 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:40,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:42,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:42,867 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:42,867 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:42,872 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:43,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:44,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:45,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:45,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:45,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:49,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:49,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:49,029 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:49,035 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:49,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:50,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:51,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:54,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:54,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:54,234 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:54,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:55,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:55,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:12:56,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:56,720 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:58,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:58,054 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:58,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:58,059 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:58,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:00,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:00,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:00,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:00,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:02,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:06,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:06,274 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:06,274 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:06,281 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:07,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:07,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:08,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:09,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:09,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:09,628 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:09,633 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:10,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:12,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:12,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:14,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:14,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:14,034 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:14,039 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:14,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:15,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:15,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:16,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:17,970 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:17,971 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:17,971 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:17,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:17,976 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:18,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:20,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:22,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:22,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:22,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:22,360 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:23,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:23,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:24,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:25,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:13:27,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:27,093 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:27,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:27,098 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:27,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:28,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:29,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:30,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:30,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:30,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:30,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:30,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:30,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:30,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:31,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:32,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:34,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:36,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:36,249 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:36,249 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:36,254 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:37,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:38,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:40,254 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:41,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:41,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:41,614 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:41,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:42,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:44,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:45,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:45,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:45,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:46,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:46,156 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:46,156 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:46,161 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:47,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:48,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:51,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:51,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:51,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:51,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:51,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:52,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:54,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:55,723 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:13:56,483 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:56,484 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:56,484 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:56,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:56,489 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:57,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:58,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:00,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:00,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:00,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:02,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:03,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:03,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:03,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:03,468 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:04,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:06,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:07,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:07,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:07,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:07,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:07,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:08,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:10,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:12,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:12,440 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:12,440 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:12,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:13,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:13,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:14,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:15,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:17,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:17,477 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:17,477 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:17,482 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:18,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:18,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:20,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:23,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:24,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:24,014 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:24,014 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:24,019 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:24,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:25,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:14:26,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:27,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:27,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:27,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:27,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:28,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:28,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:30,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:30,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:30,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:30,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:31,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:31,310 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:31,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:31,315 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:32,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:32,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:34,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:34,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:34,940 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:34,940 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:34,945 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:35,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:36,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:39,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:39,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:39,210 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:39,215 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:40,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:40,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:40,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:42,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:42,960 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:42,960 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:42,966 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:43,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:44,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:45,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:45,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:50,392 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:52,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:52,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:52,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:52,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:53,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:54,225 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:55,730 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:14:55,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:58,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:59,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:59,000 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:59,005 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:59,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:00,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:00,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:00,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:00,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:01,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:02,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:02,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:02,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:02,773 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:03,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:04,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:06,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:07,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:07,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:07,142 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:07,147 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:07,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:08,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:10,867 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:10,868 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:10,868 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:10,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:11,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:11,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:12,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:14,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:14,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:14,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:14,630 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:15,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:15,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:16,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:17,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:18,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:18,966 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:18,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:18,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:19,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:20,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:23,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:23,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:23,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:23,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:23,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:23,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:24,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:25,734 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:15:26,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:26,873 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:26,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:26,879 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:27,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:28,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:28,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:30,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:30,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:30,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:32,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:32,037 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:32,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:32,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:32,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:34,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:34,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:36,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:36,734 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:36,735 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:36,740 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:37,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:38,269 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:39,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:40,482 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:40,483 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:40,483 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:40,487 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:41,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:42,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:44,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:44,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:44,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:44,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:45,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:45,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:45,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:46,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:49,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:49,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:49,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:49,312 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:50,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:50,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:51,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:54,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:54,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:54,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:54,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:55,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:55,737 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:15:56,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:56,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:58,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:58,930 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:58,930 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:58,935 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:59,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:00,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:00,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:00,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:00,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:02,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:03,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:03,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:03,679 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:03,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:04,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:06,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:07,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:07,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:07,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:07,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:07,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:08,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:10,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:11,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:11,423 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:11,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:11,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:12,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:12,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:13,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:15,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:15,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:15,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:15,414 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:15,414 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:15,419 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:16,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:16,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:18,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:19,254 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:19,255 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:19,256 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:19,260 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:19,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:20,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:23,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:23,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:23,233 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:23,237 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:23,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:24,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:24,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:25,740 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:16:27,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:27,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:27,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:27,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:28,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:29,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:30,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:30,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:30,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:30,328 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:32,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:32,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:32,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:32,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:33,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:34,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:34,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:38,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:38,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:38,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:38,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:39,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:39,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:40,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:42,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:42,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:42,544 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:42,549 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:43,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:44,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:45,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:45,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:45,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:48,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:48,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:48,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:48,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:49,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:50,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:50,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:52,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:52,926 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:52,926 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:52,931 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:53,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:54,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:55,742 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:16:56,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:57,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:57,743 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:57,743 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:57,748 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:58,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:00,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:00,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:00,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:00,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:01,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:01,565 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:01,566 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:01,570 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:02,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:02,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:04,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:05,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:05,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:05,555 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:05,560 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:06,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:08,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:08,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:09,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:09,738 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:09,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:09,743 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:10,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:12,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:13,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:14,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:14,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:14,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:14,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:14,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:15,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:15,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:15,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:16,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:18,284 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:18,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:18,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:18,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:18,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:19,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:20,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:22,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:22,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:22,614 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:22,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:23,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:24,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:24,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:25,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:17:29,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:30,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:30,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:30,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:30,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:30,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:30,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:30,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:30,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:32,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:35,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:36,958 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:36,959 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:36,959 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:36,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:37,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:38,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:41,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:41,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:41,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:41,214 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:41,219 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:41,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:42,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:45,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:45,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:45,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:45,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:45,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:45,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:45,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:45,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:46,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:46,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:51,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:51,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:51,620 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:51,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:51,625 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:52,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:54,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:55,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:17:56,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:58,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:58,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:58,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:58,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:59,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:00,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:00,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:00,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:00,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:02,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:03,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:03,582 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:03,582 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:03,587 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:04,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:06,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:07,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:07,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:07,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:07,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:07,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:08,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:10,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:12,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:12,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:12,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:12,755 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:12,760 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:13,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:14,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:15,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:15,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:15,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:16,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:16,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:16,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:16,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:16,438 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:18,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:18,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:20,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:20,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:20,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:20,011 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:20,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:22,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:24,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:25,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:25,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:25,660 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:25,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:25,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:18:26,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:28,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:29,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:29,743 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:29,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:29,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:29,748 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:30,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:30,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:30,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:30,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:32,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:33,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:33,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:33,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:33,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:34,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:35,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:36,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:39,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:39,014 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:39,014 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:39,019 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:39,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:40,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:41,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:45,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:45,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:45,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:46,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:47,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:47,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:47,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:47,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:48,477 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:48,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:51,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:54,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:54,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:54,031 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:54,036 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:54,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:55,753 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:18:56,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:56,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:00,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:00,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:00,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:02,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:02,520 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:02,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:02,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:02,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:03,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:04,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:06,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:06,274 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:06,275 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:06,279 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:06,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:08,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:08,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:11,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:11,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:11,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:11,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:11,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:12,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:14,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:15,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:15,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:15,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:15,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:15,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:15,518 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:15,523 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:16,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:16,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:19,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:19,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:19,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:19,875 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:19,880 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:20,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:22,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:24,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:24,406 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:24,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:24,411 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:24,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:25,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:25,756 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:19:26,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:27,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:27,858 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:27,859 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:27,864 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:28,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:30,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:30,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:30,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:30,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:31,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:33,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:33,576 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:33,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:33,580 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:34,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:34,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:36,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:39,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:39,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:39,942 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:39,947 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:40,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:41,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:42,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:43,748 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:43,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:43,749 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:43,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:44,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:45,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:45,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:45,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:46,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:47,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:52,180 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:52,180 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:52,181 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:52,185 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:52,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:53,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:54,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:55,759 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:19:57,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:57,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:57,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:57,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:58,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:58,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:58,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:00,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:00,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:00,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:01,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:01,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:01,279 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:01,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:01,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:02,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:04,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:06,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:06,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:06,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:06,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:07,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:08,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:09,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:10,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:10,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:10,660 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:10,665 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:11,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:12,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:15,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:15,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:15,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:15,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:19,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:19,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:19,246 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:19,251 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:19,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:20,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:20,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:25,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:25,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:25,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:25,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:25,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:25,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:25,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:20:26,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:29,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:29,015 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:29,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:29,020 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:29,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:30,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:30,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:30,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:30,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:30,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:35,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:35,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:35,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:35,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:35,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:35,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:36,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:39,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:39,950 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:39,950 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:39,955 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:40,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:40,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:42,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:44,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:44,227 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:44,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:44,232 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:44,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:45,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:45,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:45,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:46,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:46,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:51,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:52,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:52,785 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:52,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:52,791 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:53,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:54,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:55,765 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:20:56,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:57,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:57,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:57,173 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:57,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:57,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:58,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:00,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:00,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:00,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:00,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:00,934 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:00,934 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:00,939 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:01,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:01,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:02,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:05,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:05,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:05,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:05,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:05,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:06,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:07,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:08,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:08,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:08,835 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:08,840 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:09,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:10,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:12,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:12,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:12,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:12,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:12,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:13,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:14,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:15,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:15,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:15,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:16,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:16,632 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:16,633 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:16,637 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:17,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:18,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:18,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:20,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:20,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:20,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:20,799 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:21,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:22,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:23,800 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:25,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:25,047 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:25,047 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:25,052 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:25,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:25,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:21:26,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:29,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:29,352 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:29,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:29,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:29,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:29,642 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:30,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:30,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:30,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:30,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:33,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:33,166 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:33,166 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:33,171 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:33,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:34,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:35,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:39,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:39,450 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:39,450 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:39,455 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:39,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:40,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:40,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:44,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:44,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:44,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:44,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:44,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:45,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:45,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:46,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:46,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:49,409 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:49,410 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:49,410 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:49,415 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:49,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:50,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:51,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:53,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:53,523 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:53,523 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:53,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:53,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:54,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:55,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:21:56,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:57,810 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:57,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:57,811 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:57,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:58,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:58,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:00,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:00,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:00,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:01,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:01,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:01,813 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:01,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:01,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:02,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:02,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:05,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:05,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:05,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:05,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:05,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:06,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:07,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:10,192 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:10,193 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:10,193 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:10,198 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:10,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:12,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:13,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:14,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:14,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:14,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:14,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:14,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:15,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:15,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:15,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:16,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:18,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:18,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:18,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:18,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:18,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:18,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:20,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:23,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:24,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:24,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:24,288 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:24,293 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:24,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:25,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:22:26,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:28,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:29,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:29,277 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:29,277 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:29,281 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:29,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:30,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:30,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:30,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:30,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:32,911 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:32,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:32,912 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:32,917 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:33,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:33,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:34,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:37,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:37,898 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:37,899 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:37,904 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:38,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:38,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:39,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:43,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:43,066 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:43,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:43,071 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:43,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:44,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:45,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:48,398 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:48,399 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:48,399 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:48,404 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:48,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:50,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:50,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:52,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:52,484 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:52,485 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:52,489 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:52,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:54,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:55,490 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:55,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:22:56,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:56,570 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:56,571 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:56,576 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:56,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:58,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:00,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:00,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:00,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:00,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:00,501 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:00,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:00,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:00,506 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:00,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:02,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:05,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:06,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:06,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:06,353 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:06,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:06,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:08,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:10,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:10,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:10,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:10,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:10,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:11,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:12,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:13,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:13,980 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:13,981 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:13,985 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:14,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:15,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:15,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:15,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:16,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:16,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:19,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:19,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:19,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:19,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:19,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:20,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:21,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:23,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:23,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:23,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:23,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:23,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:24,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:25,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:23:26,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:28,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:28,200 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:28,200 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:28,205 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:28,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:30,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:30,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:30,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:30,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:32,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:32,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:32,963 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:32,963 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:32,969 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:33,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:34,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:36,803 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:36,804 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:36,804 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:36,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:37,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:37,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:38,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:41,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:41,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:41,776 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:41,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:41,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:42,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:43,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:45,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:47,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:47,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:47,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:47,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:47,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:48,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:49,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:51,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:51,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:51,353 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:51,358 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:51,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:52,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:55,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:55,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:23:59,792 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:59,792 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:59,793 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:59,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:59,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:00,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:00,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:00,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:00,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:01,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:06,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:06,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:06,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:06,727 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:06,732 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:06,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:08,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:10,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:10,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:10,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:10,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:10,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:11,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:12,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:15,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:15,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:15,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:16,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:18,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:18,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:18,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:18,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:18,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:20,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:21,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:23,548 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:23,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:23,549 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:23,554 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:23,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:24,839 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:25,785 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:24:27,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:27,134 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:27,134 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:27,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:27,139 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:27,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:28,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:30,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:30,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:30,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:31,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:31,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:31,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:31,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:31,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:32,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:32,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:36,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:36,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:36,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:36,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:36,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:38,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:38,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:41,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:41,126 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:41,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:41,131 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:41,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:42,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:44,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:45,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:45,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:46,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:46,940 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:46,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:46,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:47,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:48,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:49,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:51,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:51,094 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:51,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:51,099 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:51,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:52,870 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:55,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:55,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:24:58,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:58,267 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:58,267 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:58,272 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:58,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:00,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:00,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:00,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:00,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:00,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:01,894 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:01,895 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:01,895 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:01,900 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:02,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:02,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:05,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:05,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:05,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:05,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:05,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:05,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:06,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:10,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:13,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:13,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:13,997 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:14,003 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:14,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:14,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:15,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:15,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:15,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:16,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:19,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:19,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:19,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:19,729 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:19,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:20,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:21,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:23,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:23,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:23,554 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:23,559 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:23,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:24,902 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:25,791 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:25:26,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:27,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:27,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:27,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:27,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:27,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:28,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:30,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:30,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:30,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:32,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:32,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:32,136 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:32,136 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:32,141 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:32,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:34,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:36,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:36,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:36,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:36,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:36,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:37,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:38,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:41,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:41,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:41,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:41,689 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:41,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:42,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:42,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:45,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:45,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:45,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:46,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:46,779 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:46,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:46,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:46,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:47,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:48,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:50,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:50,415 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:50,415 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:50,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:50,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:52,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:53,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:54,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:54,245 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:54,246 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:54,251 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:54,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:55,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:25:56,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:58,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:58,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:58,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:58,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:58,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:58,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:00,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:00,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:00,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:00,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:02,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:04,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:04,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:04,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:04,790 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:04,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:06,616 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:06,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:06,617 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:06,622 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:06,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:06,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:08,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:10,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:10,559 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:10,559 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:10,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:10,564 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:10,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:12,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:14,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:14,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:14,640 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:14,645 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:14,959 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:15,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:15,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:15,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:16,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:16,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:18,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:18,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:18,477 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:18,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:18,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:20,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:21,482 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:25,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:25,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:25,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:25,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:25,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:26:25,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:26,798 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:26,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:29,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:29,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:29,865 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:29,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:29,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:30,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:30,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:30,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:30,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:34,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:34,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:34,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:34,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:34,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:34,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:36,982 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:38,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:38,501 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:38,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:38,506 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:38,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:40,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:40,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:44,182 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:44,183 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:44,183 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:44,188 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:44,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:44,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:45,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:45,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:45,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:46,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:48,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:48,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:48,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:48,554 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:48,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:50,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:51,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:52,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:52,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:52,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:52,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:52,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:55,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:55,799 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:26:56,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:56,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:56,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:56,331 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:57,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:57,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:59,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:00,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:00,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:00,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:00,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:00,466 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:00,466 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:00,470 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:01,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:02,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:03,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:04,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:04,042 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:04,042 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:04,047 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:05,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:05,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:07,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:07,963 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:07,963 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:07,963 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:07,968 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:08,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:09,021 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:11,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:11,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:11,695 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:11,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:12,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:13,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:13,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:15,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:15,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:15,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:15,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:15,381 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:15,381 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:15,386 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:16,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:17,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:19,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:23,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:23,532 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:23,532 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:23,538 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:24,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:24,539 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:25,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:25,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:27:27,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:27,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:27,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:27,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:28,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:29,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:29,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:30,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:30,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:30,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:32,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:32,479 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:32,480 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:32,484 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:33,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:35,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:35,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:36,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:36,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:36,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:36,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:37,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:37,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:39,870 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:39,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:39,870 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:39,875 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:40,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:40,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:41,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:44,064 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:44,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:44,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:45,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:45,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:45,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:45,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:45,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:46,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:50,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:50,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:50,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:50,420 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:51,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:51,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:53,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:54,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:54,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:54,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:54,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:55,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:55,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:55,804 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:27:56,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:59,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:59,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:59,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:59,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:00,075 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:00,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:00,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:00,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:01,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:02,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:03,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:03,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:03,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:03,373 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:04,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:05,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:07,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:07,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:07,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:07,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:07,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:08,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:09,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:12,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:12,582 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:12,582 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:12,587 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:13,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:13,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:15,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:15,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:15,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:15,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:16,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:16,722 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:16,722 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:16,727 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:17,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:18,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:19,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:20,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:20,717 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:20,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:20,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:21,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:23,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:24,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:25,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:28:26,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:26,781 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:26,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:26,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:27,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:29,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:30,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:30,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:30,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:31,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:31,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:31,291 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:31,296 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:32,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:33,113 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:35,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:35,764 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:35,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:35,765 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:35,770 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:36,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:37,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:40,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:40,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:40,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:40,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:40,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:41,027 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:41,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:45,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:45,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:45,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:46,010 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:46,010 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:46,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:46,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:46,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:46,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:47,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:50,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:50,398 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:50,398 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:50,403 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:51,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:51,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:53,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:54,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:54,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:54,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:54,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:54,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:55,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:55,811 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:28:56,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:58,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:58,027 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:58,027 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:58,032 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:58,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:59,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:00,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:00,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:00,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:01,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:01,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:01,796 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:01,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:01,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:02,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:03,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:05,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:05,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:05,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:05,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:06,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:07,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:07,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:11,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:11,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:11,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:11,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:12,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:12,628 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:13,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:15,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:15,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:15,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:16,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:16,098 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:16,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:16,103 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:16,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:17,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:18,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:20,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:20,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:20,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:20,515 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:21,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:23,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:23,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:25,816 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:29:28,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:29,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:29,030 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:29,030 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:29,035 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:29,173 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:30,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:30,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:30,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:31,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:32,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:32,329 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:32,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:32,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:33,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:33,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:34,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:36,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:36,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:36,373 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:36,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:37,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:37,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:39,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:40,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:40,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:40,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:40,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:40,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:41,187 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:44,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:44,160 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:44,161 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:44,166 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:44,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:45,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:45,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:45,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:48,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:48,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:48,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:48,998 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:49,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:51,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:51,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:54,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:54,082 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:54,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:54,087 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:54,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:55,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:55,817 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:29:56,819 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:57,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:57,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:57,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:57,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:58,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:59,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:00,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:00,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:00,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:02,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:06,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:06,474 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:06,474 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:06,479 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:07,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:07,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:09,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:12,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:12,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:12,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:12,298 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:13,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:13,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:13,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:15,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:15,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:15,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:16,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:16,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:16,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:16,227 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:16,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:17,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:19,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:20,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:20,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:20,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:20,390 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:21,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:21,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:24,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:25,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:25,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:25,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:25,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:25,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:30:26,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:27,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:29,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:29,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:29,946 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:29,947 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:29,952 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:30,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:30,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:30,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:30,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:31,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:33,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:33,218 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:33,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:33,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:33,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:35,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:35,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:37,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:37,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:37,394 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:37,399 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:38,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:39,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:40,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:41,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:41,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:41,614 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:41,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:42,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:43,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:45,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:45,707 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:48,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:48,094 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:48,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:48,099 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:48,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:49,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:51,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:51,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:51,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:51,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:51,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:52,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:53,269 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:55,724 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:55,725 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:55,725 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:55,730 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:55,821 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:30:56,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:56,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:57,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:59,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:59,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:59,742 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:59,747 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:00,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:00,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:00,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:00,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:01,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:02,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:04,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:04,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:04,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:04,504 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:05,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:07,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:07,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:08,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:08,526 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:08,526 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:08,531 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:09,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:11,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:12,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:12,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:12,695 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:12,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:12,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:13,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:15,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:15,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:15,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:15,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:18,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:21,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:21,223 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:21,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:21,228 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:21,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:23,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:24,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:24,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:24,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:24,651 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:24,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:25,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:25,827 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:31:27,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:28,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:28,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:28,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:28,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:29,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:29,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:30,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:30,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:30,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:31,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:35,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:35,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:35,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:35,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:35,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:36,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:37,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:39,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:39,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:39,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:39,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:39,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:41,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:41,321 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:43,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:43,440 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:43,440 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:43,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:44,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:45,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:45,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:45,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:46,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:50,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:50,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:50,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:50,325 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:50,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:51,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:52,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:55,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:55,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:55,291 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:55,296 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:55,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:55,826 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:31:57,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:57,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:59,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:59,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:59,256 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:59,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:59,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:00,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:00,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:00,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:01,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:03,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:03,325 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:03,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:03,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:03,330 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:03,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:05,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:08,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:08,011 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:08,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:08,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:08,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:09,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:09,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:11,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:11,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:11,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:11,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:11,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:13,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:14,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:15,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:15,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:15,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:15,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:15,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:15,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:17,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:19,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:19,868 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:19,869 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:19,869 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:19,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:20,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:21,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:23,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:23,500 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:23,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:23,505 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:24,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:24,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:25,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:25,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:32:28,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:28,155 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:28,155 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:28,160 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:28,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:29,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:30,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:30,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:35,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:35,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:35,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:35,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:35,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:35,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:37,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:40,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:41,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:41,495 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:41,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:41,500 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:42,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:43,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:45,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:45,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:45,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:45,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:45,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:45,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:45,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:46,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:47,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:49,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:49,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:49,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:49,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:49,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:51,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:51,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:54,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:54,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:54,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:54,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:55,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:55,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:32:56,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:57,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:58,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:58,668 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:58,668 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:58,674 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:59,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:00,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:00,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:00,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:01,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:02,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:06,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:06,968 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:06,968 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:06,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:07,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:07,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:09,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:12,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:12,055 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:12,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:12,060 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:12,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:13,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:13,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:15,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:16,868 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:16,868 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:16,869 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:16,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:17,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:18,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:19,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:23,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:23,757 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:23,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:23,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:24,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:24,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:25,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:25,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:33:29,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:30,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:30,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:30,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:30,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:30,504 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:31,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:31,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:34,900 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:34,901 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:34,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:34,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:34,906 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:35,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:37,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:39,907 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:40,460 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:40,461 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:40,461 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:40,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:41,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:41,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:45,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:45,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:45,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:45,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:45,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:45,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:45,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:46,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:47,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:49,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:49,609 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:49,610 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:49,615 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:50,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:50,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:51,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:54,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:54,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:54,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:54,210 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:54,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:55,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:55,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:33:55,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:59,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:59,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:59,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:59,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:59,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:00,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:00,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:00,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:01,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:01,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:03,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:03,466 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:03,466 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:03,471 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:03,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:05,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:06,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:08,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:08,543 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:08,543 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:08,548 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:09,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:09,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:11,548 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:15,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:17,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:17,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:17,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:17,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:17,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:18,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:19,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:21,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:21,818 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:21,818 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:21,823 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:22,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:22,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:23,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:25,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:34:26,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:26,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:26,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:26,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:27,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:27,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:28,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:30,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:31,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:31,939 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:31,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:31,944 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:32,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:33,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:33,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:38,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:39,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:39,230 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:39,230 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:39,235 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:39,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:41,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:43,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:43,779 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:43,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:43,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:44,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:45,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:45,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:48,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:48,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:48,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:48,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:48,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:49,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:50,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:53,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:53,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:53,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:53,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:54,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:55,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:55,847 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:34:55,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:58,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:58,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:58,144 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:58,149 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:58,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:59,536 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:00,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:00,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:00,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:01,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:02,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:02,410 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:02,411 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:02,415 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:02,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:03,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:05,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:05,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:05,778 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:05,783 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:06,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:06,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:07,546 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:11,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:11,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:11,798 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:11,798 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:11,803 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:12,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:13,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:15,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:15,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:17,281 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:17,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:17,328 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:17,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:17,333 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:17,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:19,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:21,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:21,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:21,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:21,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:21,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:23,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:23,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:24,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:24,819 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:24,819 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:24,824 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:25,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:25,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:35:27,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:28,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:28,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:28,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:28,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:28,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:29,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:29,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:30,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:30,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:32,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:32,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:32,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:32,298 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:32,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:33,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:34,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:39,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:41,482 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:41,483 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:41,483 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:41,488 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:41,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:43,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:44,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:45,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:49,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:49,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:49,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:49,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:49,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:50,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:51,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:54,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:55,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:35:58,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:58,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:58,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:58,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:58,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:59,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:00,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:00,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:00,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:02,811 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:02,811 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:02,811 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:02,816 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:03,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:05,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:05,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:09,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:09,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:09,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:09,087 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:09,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:11,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:11,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:14,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:14,321 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:14,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:14,327 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:14,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:15,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:15,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:15,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:15,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:16,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:19,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:19,166 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:19,166 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:19,171 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:19,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:21,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:22,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:23,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:23,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:23,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:23,320 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:23,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:25,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:25,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:36:27,856 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:28,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:28,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:28,370 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:28,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:28,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:29,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:30,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:30,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:30,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:32,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:32,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:32,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:32,746 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:33,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:33,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:33,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:38,648 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:38,649 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:38,650 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:38,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:39,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:39,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:39,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:42,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:42,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:42,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:42,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:43,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:44,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:45,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:45,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:47,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:47,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:47,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:47,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:48,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:49,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:50,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:52,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:52,862 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:52,862 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:52,867 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:53,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:53,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:55,857 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:36:56,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:57,875 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:57,876 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:57,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:57,881 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:58,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:59,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:00,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:00,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:02,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:03,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:03,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:03,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:03,698 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:04,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:05,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:07,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:07,450 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:07,451 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:07,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:07,456 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:07,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:09,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:12,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:12,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:12,517 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:12,517 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:12,522 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:12,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:13,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:15,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:15,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:15,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:17,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:17,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:17,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:17,724 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:17,729 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:18,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:19,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:22,530 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:22,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:22,532 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:22,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:22,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:23,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:23,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:25,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:37:26,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:26,722 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:26,722 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:26,727 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:27,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:27,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:28,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:30,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:30,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:30,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:30,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:30,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:30,387 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:30,392 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:30,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:31,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:34,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:35,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:35,406 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:35,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:35,411 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:35,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:37,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:39,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:39,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:39,601 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:39,602 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:39,606 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:39,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:41,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:43,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:43,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:43,422 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:43,426 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:43,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:45,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:45,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:45,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:48,450 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:48,450 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:48,450 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:48,455 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:48,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:49,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:50,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:55,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:55,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:37:56,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:56,975 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:56,976 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:56,980 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:57,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:59,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:00,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:00,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:00,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:00,502 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:00,502 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:00,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:00,507 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:00,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:01,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:04,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:04,382 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:04,383 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:04,388 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:04,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:05,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:06,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:08,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:08,844 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:08,844 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:08,850 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:09,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:09,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:11,850 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:15,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:15,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:15,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:17,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:17,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:17,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:17,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:17,278 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:17,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:19,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:22,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:23,267 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:23,268 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:23,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:23,274 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:23,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:25,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:25,869 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:38:27,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:27,735 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:27,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:27,736 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:27,741 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:27,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:29,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:30,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:30,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:30,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:32,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:32,578 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:32,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:32,583 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:32,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:33,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:33,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:38,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:41,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:41,168 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:41,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:41,173 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:41,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:43,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:44,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:45,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:45,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:45,878 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:45,878 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:45,883 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:46,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:47,789 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:49,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:51,928 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:51,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:51,929 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:51,934 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:52,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:53,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:54,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:55,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:38:56,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:56,878 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:56,879 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:56,883 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:57,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:57,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:00,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:00,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:00,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:00,472 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:00,472 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:00,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:00,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:01,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:04,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:04,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:04,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:04,377 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:04,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:05,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:05,809 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:08,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:08,535 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:08,535 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:08,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:08,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:09,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:10,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:13,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:13,787 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:13,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:13,792 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:13,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:15,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:15,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:15,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:15,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:16,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:17,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:17,821 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:17,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:17,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:18,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:19,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:21,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:21,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:21,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:21,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:21,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:21,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:23,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:25,727 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:25,728 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:25,728 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:25,732 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:25,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:25,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:39:26,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:27,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:30,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:30,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:30,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:30,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:30,299 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:30,299 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:30,304 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:30,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:31,839 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:32,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:37,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:37,168 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:37,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:37,173 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:37,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:38,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:39,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:41,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:41,181 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:41,182 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:41,187 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:41,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:43,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:43,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:44,811 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:44,812 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:44,812 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:44,817 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:44,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:45,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:45,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:45,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:48,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:53,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:54,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:54,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:54,532 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:54,537 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:54,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:55,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:55,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:39:58,875 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:00,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:00,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:00,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:00,673 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:00,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:00,678 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:00,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:01,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:04,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:04,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:04,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:04,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:04,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:04,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:05,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:09,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:10,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:10,062 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:10,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:10,067 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:10,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:11,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:14,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:14,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:14,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:14,059 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:14,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:15,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:15,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:15,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:15,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:15,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:18,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:18,639 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:18,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:18,644 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:18,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:19,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:20,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:22,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:22,047 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:22,047 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:22,053 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:22,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:23,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:25,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:40:25,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:27,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:27,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:27,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:27,341 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:27,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:29,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:30,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:30,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:30,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:31,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:31,675 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:31,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:31,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:31,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:31,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:33,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:36,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:36,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:36,814 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:36,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:36,819 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:36,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:37,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:40,668 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:40,668 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:40,669 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:40,673 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:40,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:41,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:42,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:44,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:44,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:44,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:44,309 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:44,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:45,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:45,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:45,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:48,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:48,088 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:48,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:48,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:48,094 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:48,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:49,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:53,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:54,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:54,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:54,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:54,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:54,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:55,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:40:55,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:58,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:00,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:00,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:00,563 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:00,563 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:00,564 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:00,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:00,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:01,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:04,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:05,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:05,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:05,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:05,610 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:05,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:07,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:09,611 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:10,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:10,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:10,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:10,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:10,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:11,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:14,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:15,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:15,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:15,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:15,808 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:15,809 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:15,809 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:15,814 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:15,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:17,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:19,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:19,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:19,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:19,610 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:19,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:20,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:21,959 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:24,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:24,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:24,628 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:24,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:24,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:25,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:25,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:41:25,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:30,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:30,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:30,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:30,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:30,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:30,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:30,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:30,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:31,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:31,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:34,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:34,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:34,666 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:34,671 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:34,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:35,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:36,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:38,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:38,127 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:38,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:38,132 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:38,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:39,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:42,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:44,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:44,489 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:44,490 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:44,495 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:44,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:45,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:45,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:45,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:45,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:47,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:48,670 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:48,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:48,671 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:48,676 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:48,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:49,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:52,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:52,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:52,286 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:52,291 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:53,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:53,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:54,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:55,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:55,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:55,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:55,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:55,884 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:41:56,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:58,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:58,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:59,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:59,916 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:59,916 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:59,921 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:00,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:00,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:00,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:00,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:02,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:03,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:03,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:03,517 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:03,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:04,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:04,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:06,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:08,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:08,429 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:08,429 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:08,434 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:09,021 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:10,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:10,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:12,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:12,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:12,627 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:12,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:13,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:14,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:15,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:15,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:15,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:16,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:16,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:16,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:16,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:16,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:17,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:18,033 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:20,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:20,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:20,508 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:20,513 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:21,035 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:21,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:22,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:24,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:24,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:24,695 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:24,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:25,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:25,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:42:26,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:26,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:29,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:29,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:29,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:29,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:30,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:30,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:30,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:30,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:30,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:32,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:36,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:36,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:36,682 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:36,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:37,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:37,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:38,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:41,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:41,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:41,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:41,298 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:42,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:43,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:44,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:45,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:45,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:45,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:48,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:49,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:49,800 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:49,800 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:49,805 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:50,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:52,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:53,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:54,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:54,270 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:54,270 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:54,275 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:55,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:55,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:42:56,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:58,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:58,911 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:58,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:58,912 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:58,917 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:59,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:00,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:00,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:00,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:00,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:03,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:03,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:03,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:03,328 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:04,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:04,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:06,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:09,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:09,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:09,846 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:09,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:09,852 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:10,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:12,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:13,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:13,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:13,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:13,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:14,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:14,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:15,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:15,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:15,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:16,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:18,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:18,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:18,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:18,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:19,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:20,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:20,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:21,764 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:21,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:21,765 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:21,770 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:22,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:24,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:25,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:25,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:43:26,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:26,354 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:26,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:26,359 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:27,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:28,106 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:29,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:29,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:29,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:29,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:30,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:30,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:30,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:30,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:31,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:32,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:36,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:37,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:37,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:37,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:37,321 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:38,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:40,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:41,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:41,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:41,288 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:41,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:42,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:42,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:42,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:45,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:45,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:45,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:45,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:45,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:45,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:45,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:46,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:47,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:49,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:49,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:49,774 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:49,779 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:50,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:52,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:52,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:54,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:54,916 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:54,917 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:54,921 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:55,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:55,898 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:43:56,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:57,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:59,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:59,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:59,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:59,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:00,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:00,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:00,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:00,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:02,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:03,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:03,997 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:03,998 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:03,998 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:04,003 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:04,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:06,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:07,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:07,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:07,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:07,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:08,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:08,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:10,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:11,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:11,781 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:11,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:11,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:12,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:13,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:14,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:15,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:15,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:15,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:19,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:20,192 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:20,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:20,193 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:20,197 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:21,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:22,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:25,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:25,897 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:44:26,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:26,033 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:26,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:26,038 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:26,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:28,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:30,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:30,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:30,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:30,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:31,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:31,564 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:31,565 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:31,569 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:32,173 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:34,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:35,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:35,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:35,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:35,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:35,377 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:36,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:36,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:39,210 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:39,211 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:39,211 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:39,216 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:40,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:40,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:41,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:45,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:45,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:45,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:46,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:48,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:48,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:48,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:48,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:48,189 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:50,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:52,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:52,675 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:52,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:52,676 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:52,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:53,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:54,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:55,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:44:57,904 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:58,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:58,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:58,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:58,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:59,199 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:00,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:00,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:00,201 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:02,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:02,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:02,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:02,191 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:02,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:03,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:04,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:08,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:10,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:10,478 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:10,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:10,483 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:11,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:12,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:13,483 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:15,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:15,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:15,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:19,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:19,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:19,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:19,679 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:19,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:20,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:22,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:23,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:23,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:23,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:23,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:23,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:24,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:25,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:25,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:45:26,751 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:26,752 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:26,752 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:26,757 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:27,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:28,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:30,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:30,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:30,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:30,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:31,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:31,289 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:31,290 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:31,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:32,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:32,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:35,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:35,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:35,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:35,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:35,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:36,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:36,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:40,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:40,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:40,889 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:40,889 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:40,894 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:41,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:42,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:44,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:44,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:44,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:44,649 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:45,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:45,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:45,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:45,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:46,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:46,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:51,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:52,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:52,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:52,900 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:52,905 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:53,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:54,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:55,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:45:56,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:56,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:56,666 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:56,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:56,671 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:57,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:58,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:00,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:01,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:01,328 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:01,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:01,333 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:02,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:02,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:02,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:05,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:05,367 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:05,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:05,372 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:06,269 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:06,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:08,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:10,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:10,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:10,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:10,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:10,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:12,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:14,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:15,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:15,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:15,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:16,872 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:16,872 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:16,872 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:16,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:17,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:18,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:19,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:20,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:20,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:20,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:20,748 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:21,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:22,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:25,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:25,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:46:28,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:28,988 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:28,988 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:28,993 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:29,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:30,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:30,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:30,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:30,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:31,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:34,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:34,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:34,009 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:34,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:34,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:36,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:37,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:37,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:37,957 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:37,958 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:37,962 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:38,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:40,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:41,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:41,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:41,927 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:41,932 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:42,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:42,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:44,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:45,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:45,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:45,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:45,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:45,475 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:45,475 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:45,480 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:46,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:46,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:48,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:51,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:51,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:51,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:51,279 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:51,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:52,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:54,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:55,913 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:46:59,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:59,748 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:59,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:59,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:59,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:00,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:00,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:02,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:05,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:08,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:08,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:08,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:08,119 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:08,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:10,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:11,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:12,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:12,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:12,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:12,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:12,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:14,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:15,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:15,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:15,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:16,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:16,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:16,985 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:16,985 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:16,990 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:17,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:18,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:22,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:22,530 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:22,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:22,531 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:22,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:23,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:24,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:25,916 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:47:26,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:26,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:26,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:26,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:27,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:27,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:28,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:30,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:30,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:30,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:30,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:30,781 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:30,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:30,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:31,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:32,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:32,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:36,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:36,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:36,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:36,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:37,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:38,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:38,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:40,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:40,699 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:40,699 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:40,704 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:41,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:42,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:43,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:44,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:44,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:44,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:44,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:45,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:45,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:45,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:45,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:46,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:48,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:48,597 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:48,597 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:48,602 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:49,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:49,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:50,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:52,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:52,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:52,678 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:52,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:53,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:54,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:54,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:55,918 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:47:56,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:56,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:56,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:56,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:57,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:58,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:00,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:02,675 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:02,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:02,676 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:02,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:03,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:04,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:05,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:06,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:06,723 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:06,723 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:06,728 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:07,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:08,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:10,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:15,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:15,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:15,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:15,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:15,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:15,349 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:15,354 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:15,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:16,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:16,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:19,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:19,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:19,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:19,121 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:19,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:20,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:22,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:22,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:22,716 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:22,716 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:22,721 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:23,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:24,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:25,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:48:26,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:26,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:26,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:26,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:26,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:27,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:28,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:30,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:30,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:30,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:30,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:30,380 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:30,380 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:30,385 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:30,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:32,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:32,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:35,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:35,878 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:35,878 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:35,883 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:36,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:37,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:38,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:40,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:40,257 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:40,258 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:40,262 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:40,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:42,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:43,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:45,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:45,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:45,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:45,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:45,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:45,967 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:45,972 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:46,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:48,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:49,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:49,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:49,889 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:49,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:49,894 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:50,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:52,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:53,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:53,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:53,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:53,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:54,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:54,910 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:55,922 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:48:56,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:57,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:57,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:57,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:57,621 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:58,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:58,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:00,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:00,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:02,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:02,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:02,927 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:02,933 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:03,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:04,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:05,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:08,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:08,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:08,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:08,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:09,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:10,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:11,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:15,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:15,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:15,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:17,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:18,267 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:18,268 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:18,268 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:18,273 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:18,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:20,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:21,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:21,833 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:21,833 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:21,838 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:22,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:22,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:24,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:25,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:49:26,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:26,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:26,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:26,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:27,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:27,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:28,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:30,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:30,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:30,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:31,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:31,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:31,378 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:31,383 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:31,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:32,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:33,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:35,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:35,342 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:35,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:35,347 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:35,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:36,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:39,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:39,057 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:39,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:39,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:39,062 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:39,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:40,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:44,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:44,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:44,717 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:44,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:44,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:45,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:45,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:45,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:45,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:46,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:49,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:49,289 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:49,289 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:49,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:49,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:49,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:50,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:54,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:54,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:54,142 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:54,147 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:54,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:55,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:55,926 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:49:56,511 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:57,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:57,791 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:57,792 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:57,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:58,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:00,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:00,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:00,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:02,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:02,297 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:02,297 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:02,302 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:02,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:04,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:06,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:08,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:08,632 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:08,633 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:08,638 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:09,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:10,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:11,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:12,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:12,541 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:12,541 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:12,546 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:13,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:14,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:15,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:15,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:15,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:17,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:18,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:18,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:18,209 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:18,214 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:18,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:20,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:23,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:25,931 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:50:26,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:26,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:26,566 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:26,571 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:27,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:28,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:28,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:30,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:30,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:30,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:33,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:33,950 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:33,950 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:33,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:33,955 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:34,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:36,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:37,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:37,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:37,997 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:38,002 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:38,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:39,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:40,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:44,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:44,343 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:44,344 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:44,348 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:44,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:45,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:45,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:45,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:46,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:48,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:48,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:48,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:48,895 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:49,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:49,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:50,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:52,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:52,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:52,906 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:52,911 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:53,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:54,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:55,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:55,930 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:50:56,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:56,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:56,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:56,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:57,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:58,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:00,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:00,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:01,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:03,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:03,068 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:03,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:03,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:03,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:04,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:07,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:07,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:07,459 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:07,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:07,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:07,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:08,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:12,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:12,187 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:12,187 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:12,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:12,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:12,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:14,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:15,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:15,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:15,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:16,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:16,781 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:16,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:16,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:17,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:17,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:18,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:20,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:20,577 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:20,578 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:20,583 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:20,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:22,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:23,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:25,936 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:51:26,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:26,258 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:26,258 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:26,263 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:26,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:28,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:29,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:30,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:30,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:30,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:33,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:33,408 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:33,408 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:33,413 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:33,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:34,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:34,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:38,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:38,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:38,173 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:38,178 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:38,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:40,179 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:40,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:43,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:43,504 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:43,504 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:43,509 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:43,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:44,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:45,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:45,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:45,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:45,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:48,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:48,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:48,215 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:48,220 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:48,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:50,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:51,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:52,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:52,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:52,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:52,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:52,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:54,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:55,935 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:51:56,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:56,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:56,364 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:56,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:56,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:56,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:58,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:00,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:00,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:00,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:01,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:01,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:01,546 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:01,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:01,550 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:01,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:02,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:05,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:05,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:05,682 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:05,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:06,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:06,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:06,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:09,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:09,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:09,731 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:09,736 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:10,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:10,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:11,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:13,970 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:13,971 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:13,971 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:13,976 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:14,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:15,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:15,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:15,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:16,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:17,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:21,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:21,121 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:21,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:21,126 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:21,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:22,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:23,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:25,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:25,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:25,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:25,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:25,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:25,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:52:26,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:28,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:30,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:30,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:30,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:31,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:31,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:31,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:31,768 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:32,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:32,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:34,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:36,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:36,886 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:36,886 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:36,891 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:37,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:38,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:39,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:42,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:42,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:42,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:42,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:42,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:44,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:45,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:45,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:45,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:45,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:50,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:50,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:50,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:50,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:50,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:50,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:52,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:54,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:54,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:54,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:54,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:55,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:55,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:55,938 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:52:56,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:59,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:59,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:59,370 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:59,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:59,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:00,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:00,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:00,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:00,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:01,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:03,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:03,802 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:03,803 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:03,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:04,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:04,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:06,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:08,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:08,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:08,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:08,886 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:09,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:10,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:11,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:12,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:12,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:12,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:12,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:13,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:14,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:15,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:15,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:15,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:16,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:16,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:16,395 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:16,400 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:16,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:17,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:18,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:19,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:19,730 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:19,731 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:19,740 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:20,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:20,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:22,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:24,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:24,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:24,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:24,331 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:24,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:25,940 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:53:26,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:27,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:28,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:28,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:28,281 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:28,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:28,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:30,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:30,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:30,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:30,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:32,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:32,381 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:32,381 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:32,386 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:32,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:33,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:34,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:38,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:38,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:38,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:38,660 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:38,665 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:38,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:40,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:43,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:43,523 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:43,523 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:43,523 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:43,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:43,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:44,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:45,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:45,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:45,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:47,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:47,846 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:47,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:47,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:48,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:48,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:48,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:52,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:52,667 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:52,667 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:52,672 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:52,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:54,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:54,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:55,945 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:53:57,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:57,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:57,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:57,794 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:57,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:58,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:59,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:00,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:00,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:00,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:02,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:02,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:02,505 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:02,510 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:02,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:04,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:05,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:08,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:08,589 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:08,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:08,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:08,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:10,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:10,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:15,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:15,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:15,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:16,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:17,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:17,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:17,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:17,113 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:17,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:18,824 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:21,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:21,077 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:21,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:21,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:21,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:22,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:22,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:24,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:24,840 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:24,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:24,846 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:25,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:25,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:54:26,836 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:27,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:30,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:30,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:30,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:31,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:31,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:31,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:31,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:31,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:32,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:33,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:35,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:35,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:35,855 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:35,860 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:36,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:36,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:38,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:39,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:39,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:39,975 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:39,980 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:40,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:40,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:43,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:44,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:44,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:44,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:44,157 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:44,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:45,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:45,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:45,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:46,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:47,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:47,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:47,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:47,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:47,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:48,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:49,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:52,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:52,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:52,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:52,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:52,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:54,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:55,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:55,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:54:56,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:56,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:56,353 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:56,358 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:56,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:58,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:00,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:00,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:00,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:01,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:01,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:01,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:01,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:01,795 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:01,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:02,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:06,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:07,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:07,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:07,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:07,011 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:07,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:08,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:11,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:11,423 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:11,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:11,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:11,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:12,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:12,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:15,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:15,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:15,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:15,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:15,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:15,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:15,667 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:15,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:16,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:17,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:20,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:20,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:20,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:20,515 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:20,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:22,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:23,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:24,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:24,300 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:24,301 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:24,306 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:24,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:25,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:55:26,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:28,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:28,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:28,712 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:28,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:28,717 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:28,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:30,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:30,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:30,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:30,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:32,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:32,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:32,142 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:32,147 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:32,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:34,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:34,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:39,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:40,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:40,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:40,628 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:40,636 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:40,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:42,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:44,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:45,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:45,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:45,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:45,444 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:45,445 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:45,445 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:45,450 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:45,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:46,942 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:49,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:49,845 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:49,845 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:49,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:49,850 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:49,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:52,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:54,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:54,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:54,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:54,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:54,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:55,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:55,954 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:55:56,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:58,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:58,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:58,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:58,337 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:58,954 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:00,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:00,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:00,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:00,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:00,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:02,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:02,075 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:02,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:02,081 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:02,959 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:04,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:06,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:06,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:06,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:06,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:06,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:06,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:08,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:11,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:11,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:11,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:11,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:11,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:11,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:12,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:15,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:15,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:15,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:16,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:17,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:17,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:17,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:17,109 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:17,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:18,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:21,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:21,060 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:21,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:21,065 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:21,979 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:22,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:22,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:25,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:56:26,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:26,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:26,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:26,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:26,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:27,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:28,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:30,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:30,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:30,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:30,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:30,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:30,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:30,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:30,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:32,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:32,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:34,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:34,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:34,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:34,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:34,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:36,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:38,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:38,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:38,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:38,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:38,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:38,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:40,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:43,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:43,530 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:43,530 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:43,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:43,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:44,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:45,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:45,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:45,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:45,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:49,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:49,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:49,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:49,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:49,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:50,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:51,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:54,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:54,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:54,519 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:54,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:54,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:55,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:55,958 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:56:57,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:59,960 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:00,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:00,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:00,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:01,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:01,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:01,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:01,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:02,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:03,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:05,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:06,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:06,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:06,854 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:06,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:07,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:09,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:10,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:12,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:12,371 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:12,371 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:12,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:13,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:15,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:15,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:15,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:15,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:16,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:16,488 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:16,489 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:16,489 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:16,494 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:17,035 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:19,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:20,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:20,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:20,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:20,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:21,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:21,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:23,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:23,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:23,930 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:23,930 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:23,936 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:24,043 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:25,957 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:57:26,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:27,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:30,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:30,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:30,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:32,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:32,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:32,265 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:32,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:32,270 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:33,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:35,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:35,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:35,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:35,948 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:35,953 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:36,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:37,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:39,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:40,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:40,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:40,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:40,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:41,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:43,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:43,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:45,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:45,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:45,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:46,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:46,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:46,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:46,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:47,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:49,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:49,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:51,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:51,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:51,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:51,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:51,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:53,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:55,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:55,959 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:57:57,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:57,334 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:57,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:57,340 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:58,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:59,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:00,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:00,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:00,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:00,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:02,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:02,364 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:02,365 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:02,370 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:03,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:05,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:05,409 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:09,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:09,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:09,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:09,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:10,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:10,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:11,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:13,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:13,827 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:13,827 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:13,832 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:14,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:15,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:15,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:15,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:16,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:17,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:18,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:18,067 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:18,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:18,072 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:18,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:21,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:22,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:22,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:22,383 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:22,383 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:22,388 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:23,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:25,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:25,961 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:58:27,963 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:30,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:30,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:30,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:30,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:30,851 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:30,852 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:30,856 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:31,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:33,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:33,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:35,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:35,132 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:35,132 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:35,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:36,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:37,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:38,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:38,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:38,947 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:38,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:38,952 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:39,116 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:41,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:43,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:44,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:44,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:44,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:44,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:44,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:45,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:45,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:45,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:47,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:48,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:48,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:48,420 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:48,425 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:49,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:49,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:51,128 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:52,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:52,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:52,394 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:52,399 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:53,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:55,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:55,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:55,963 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:58:56,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:56,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:56,640 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:56,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:57,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:59,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:00,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:00,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:00,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:00,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:00,597 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:00,598 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:00,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:00,603 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:01,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:03,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:04,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:04,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:04,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:04,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:05,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:06,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:07,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:08,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:08,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:08,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:08,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:09,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:11,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:12,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:12,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:12,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:12,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:12,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:13,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:15,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:15,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:15,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:15,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:16,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:16,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:16,234 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:16,239 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:17,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:18,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:19,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:23,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:24,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:24,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:24,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:24,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:25,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:25,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:59:27,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:28,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:28,092 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:28,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:28,097 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:28,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:29,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:30,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:30,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:30,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:31,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:32,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:32,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:32,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:32,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:33,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:34,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:35,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:36,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:36,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:36,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:36,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:37,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:39,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:39,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:40,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:40,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:40,690 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:40,695 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:41,180 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:43,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:45,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:45,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:45,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:45,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:45,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:45,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:45,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:45,811 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:46,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:47,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:49,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:49,208 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:49,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:49,214 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:50,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:50,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:51,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:54,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:54,426 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:54,426 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:54,431 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:55,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:55,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:55,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:59:57,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:00,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:00,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:00,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:01,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:02,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:02,791 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:02,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:02,796 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:03,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:05,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:06,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:08,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:08,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:08,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:08,956 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:09,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:11,212 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:11,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:13,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:13,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:13,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:13,230 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:14,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:15,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:15,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:15,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:15,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:17,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:17,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:17,002 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:17,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:17,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:17,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:19,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:22,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:22,968 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:22,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:22,969 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:22,974 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:23,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:25,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:25,973 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:00:26,976 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:26,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:26,977 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:26,981 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:27,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:27,982 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:29,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:30,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:30,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:30,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:30,408 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:30,409 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:30,409 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:30,414 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:31,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:33,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:33,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:38,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:38,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:38,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:38,776 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:38,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:39,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:41,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:42,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:42,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:42,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:42,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:43,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:43,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:45,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:45,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:45,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:45,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:49,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:49,041 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:49,041 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:49,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:49,046 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:49,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:51,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:54,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:55,972 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:00:57,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:57,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:57,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:57,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:58,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:59,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:59,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:00,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:00,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:00,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:02,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:02,604 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:02,604 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:02,609 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:03,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:04,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:05,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:06,442 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:06,443 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:06,443 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:06,449 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:07,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:09,269 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:10,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:11,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:11,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:11,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:11,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:12,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:13,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:15,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:15,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:15,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:16,392 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:18,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:18,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:18,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:18,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:19,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:21,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:21,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:25,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:01:26,978 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:27,048 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:27,049 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:27,049 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:27,054 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:27,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:29,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:30,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:30,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:30,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:32,250 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:32,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:32,251 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:32,251 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:32,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:32,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:35,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:36,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:36,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:36,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:36,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:36,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:38,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:39,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:42,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:42,530 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:42,530 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:42,535 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:43,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:43,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:45,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:45,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:45,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:45,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:46,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:46,131 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:46,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:46,136 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:46,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:49,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:49,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:49,670 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:49,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:49,671 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:49,676 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:50,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:51,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:54,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:55,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:55,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:55,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:55,298 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:55,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:55,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:01:57,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:59,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:00,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:00,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:00,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:01,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:01,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:01,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:01,230 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:01,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:03,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:05,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:07,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:07,387 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:07,387 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:07,392 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:08,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:09,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:10,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:11,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:11,803 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:11,803 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:11,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:12,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:13,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:15,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:15,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:15,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:16,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:21,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:21,014 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:21,014 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:21,019 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:21,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:22,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:23,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:25,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:02:27,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:29,748 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:29,748 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:29,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:29,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:30,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:30,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:30,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:30,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:31,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:33,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:35,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:35,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:35,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:35,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:36,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:37,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:38,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:39,820 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:39,821 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:39,821 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:39,826 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:40,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:41,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:43,826 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:44,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:44,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:44,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:44,845 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:45,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:45,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:45,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:45,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:47,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:49,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:49,245 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:49,246 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:49,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:49,250 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:49,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:51,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:54,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:54,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:54,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:54,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:54,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:55,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:55,979 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:02:57,375 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:58,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:58,582 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:58,582 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:58,587 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:59,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:59,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:00,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:00,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:00,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:01,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:05,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:06,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:06,568 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:06,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:06,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:07,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:09,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:10,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:11,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:11,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:11,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:11,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:12,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:13,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:15,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:15,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:15,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:16,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:16,244 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:16,244 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:16,244 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:16,249 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:16,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:19,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:19,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:19,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:19,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:19,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:20,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:21,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:21,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:23,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:23,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:23,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:23,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:24,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:25,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:25,981 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:03:26,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:29,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:29,729 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:29,729 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:29,734 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:30,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:30,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:30,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:30,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:31,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:32,392 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:35,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:35,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:35,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:35,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:36,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:37,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:37,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:40,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:40,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:40,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:40,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:40,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:41,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:43,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:45,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:45,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:45,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:45,986 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:45,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:45,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:45,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:46,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:47,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:49,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:51,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:51,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:51,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:51,144 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:51,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:53,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:54,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:55,987 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:03:57,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:57,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:57,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:57,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:58,438 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:59,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:59,751 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:00,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:00,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:00,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:03,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:03,103 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:03,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:03,108 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:03,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:05,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:05,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:06,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:06,729 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:06,729 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:06,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:07,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:09,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:10,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:15,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:15,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:15,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:15,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:15,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:15,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:15,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:15,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:16,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:17,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:20,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:20,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:20,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:20,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:21,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:21,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:23,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:25,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:25,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:25,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:25,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:25,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:04:26,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:26,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:27,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:30,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:30,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:30,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:30,274 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:30,367 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:30,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:30,372 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:30,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:32,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:33,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:34,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:34,586 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:34,586 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:34,591 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:35,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:37,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:37,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:39,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:39,088 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:39,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:39,093 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:39,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:41,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:42,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:42,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:42,879 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:42,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:42,884 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:43,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:45,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:45,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:45,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:45,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:48,426 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:49,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:49,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:49,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:49,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:50,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:51,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:53,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:54,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:54,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:54,537 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:54,542 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:55,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:55,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:04:57,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:58,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:58,715 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:58,715 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:58,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:58,721 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:59,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:00,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:00,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:00,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:01,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:02,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:02,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:02,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:02,879 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:03,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:03,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:05,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:07,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:07,834 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:07,834 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:07,839 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:08,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:09,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:09,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:12,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:12,194 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:12,194 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:12,199 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:12,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:13,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:15,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:15,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:15,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:15,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:18,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:18,019 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:18,019 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:18,024 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:18,522 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:19,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:21,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:23,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:23,170 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:23,171 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:23,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:23,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:25,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:25,990 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:05:26,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:27,647 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:27,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:27,648 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:27,653 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:28,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:29,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:30,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:30,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:30,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:31,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:31,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:31,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:31,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:32,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:32,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:33,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:37,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:38,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:38,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:38,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:38,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:38,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:39,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:41,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:41,779 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:41,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:41,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:42,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:43,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:43,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:45,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:45,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:45,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:45,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:45,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:45,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:45,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:46,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:47,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:48,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:49,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:49,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:49,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:49,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:50,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:51,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:54,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:54,103 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:54,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:54,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:54,108 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:54,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:55,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:55,995 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:05:58,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:58,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:58,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:58,312 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:58,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:59,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:59,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:00,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:00,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:00,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:02,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:02,791 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:02,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:02,796 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:03,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:04,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:05,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:06,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:06,789 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:06,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:06,795 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:07,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:09,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:10,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:11,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:11,668 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:11,668 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:11,673 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:12,578 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:13,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:15,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:15,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:15,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:16,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:17,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:17,059 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:17,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:17,064 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:17,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:19,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:21,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:21,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:21,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:21,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:21,391 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:21,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:23,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:25,994 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:06:26,995 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:27,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:27,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:27,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:27,011 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:27,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:29,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:30,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:30,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:30,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:31,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:31,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:31,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:31,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:31,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:32,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:33,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:35,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:35,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:35,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:35,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:35,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:37,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:38,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:39,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:39,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:39,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:39,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:39,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:41,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:44,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:45,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:45,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:45,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:45,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:45,484 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:45,485 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:45,490 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:45,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:47,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:49,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:51,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:51,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:51,034 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:51,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:51,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:53,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:55,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:55,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:55,449 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:55,449 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:55,455 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:55,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:55,999 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:06:57,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:59,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:59,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:59,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:59,278 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:59,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:00,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:00,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:00,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:01,635 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:04,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:04,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:04,676 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:04,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:05,637 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:05,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:07,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:09,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:09,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:09,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:09,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:09,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:11,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:11,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:14,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:14,604 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:14,604 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:14,609 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:14,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:15,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:15,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:15,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:16,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:17,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:18,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:18,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:18,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:18,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:19,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:21,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:21,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:22,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:22,886 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:22,886 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:22,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:23,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:25,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:25,998 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:07:27,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:27,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:27,119 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:27,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:27,124 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:27,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:29,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:30,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:30,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:30,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:31,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:31,071 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:31,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:31,076 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:31,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:32,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:33,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:34,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:34,451 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:34,452 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:34,457 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:34,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:37,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:37,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:38,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:38,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:38,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:38,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:39,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:41,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:42,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:43,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:43,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:43,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:43,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:43,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:45,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:45,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:45,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:45,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:47,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:47,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:47,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:47,567 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:47,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:48,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:49,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:52,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:52,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:52,215 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:52,220 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:52,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:53,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:54,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:56,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:07:57,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:57,172 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:57,173 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:57,178 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:57,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:59,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:00,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:00,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:02,027 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:02,030 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:02,030 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:02,036 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:02,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:03,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:06,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:07,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:07,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:07,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:07,747 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:08,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:09,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:11,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:11,589 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:11,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:11,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:11,594 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:11,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:13,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:15,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:15,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:15,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:15,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:15,359 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:15,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:15,364 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:15,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:17,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:17,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:19,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:19,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:19,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:19,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:19,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:21,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:22,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:23,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:23,389 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:23,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:23,394 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:23,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:25,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:26,002 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:08:26,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:26,975 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:26,975 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:26,979 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:27,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:27,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:29,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:30,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:30,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:30,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:32,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:32,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:32,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:32,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:32,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:33,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:33,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:38,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:40,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:40,208 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:40,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:40,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:40,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:41,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:43,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:44,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:44,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:44,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:44,596 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:44,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:45,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:45,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:45,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:47,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:48,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:48,175 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:48,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:48,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:48,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:48,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:49,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:53,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:56,004 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:08:57,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:57,203 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:57,203 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:57,208 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:57,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:59,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:59,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:00,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:00,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:02,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:02,458 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:02,458 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:02,463 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:02,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:03,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:04,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:05,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:05,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:05,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:05,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:06,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:07,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:09,816 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:10,192 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:10,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:10,193 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:10,198 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:10,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:11,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:14,042 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:14,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:14,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:14,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:14,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:15,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:15,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:15,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:15,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:15,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:17,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:17,819 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:17,819 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:17,824 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:18,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:19,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:20,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:25,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:25,408 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:25,408 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:25,414 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:25,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:26,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:09:26,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:27,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:30,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:30,025 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:30,025 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:30,030 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:30,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:30,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:30,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:30,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:31,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:31,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:34,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:34,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:34,875 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:34,881 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:35,790 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:36,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:37,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:38,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:38,480 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:38,480 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:38,485 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:38,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:39,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:42,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:43,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:43,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:43,018 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:43,024 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:43,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:45,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:45,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:45,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:45,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:47,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:47,122 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:47,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:47,127 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:47,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:48,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:49,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:50,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:50,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:50,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:50,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:51,809 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:53,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:53,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:54,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:54,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:54,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:54,998 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:55,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:56,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:09:57,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:59,010 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:59,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:59,833 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:59,833 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:59,838 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:00,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:00,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:01,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:03,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:03,585 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:03,585 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:03,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:03,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:04,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:05,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:07,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:07,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:07,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:07,190 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:07,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:09,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:10,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:11,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:11,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:11,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:11,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:11,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:13,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:15,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:15,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:15,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:15,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:15,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:15,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:15,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:15,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:15,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:17,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:20,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:21,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:21,004 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:21,004 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:21,009 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:21,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:23,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:26,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:26,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:26,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:10:26,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:26,013 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:26,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:26,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:27,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:30,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:30,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:30,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:31,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:31,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:31,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:31,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:31,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:31,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:33,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:34,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:34,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:34,678 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:34,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:34,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:36,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:37,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:38,968 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:38,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:38,969 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:38,974 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:39,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:41,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:41,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:42,970 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:42,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:42,971 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:42,976 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:43,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:45,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:45,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:45,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:45,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:47,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:51,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:51,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:51,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:51,365 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:51,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:52,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:53,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:56,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:10:56,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:56,221 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:56,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:56,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:56,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:57,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:58,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:00,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:00,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:00,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:00,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:00,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:00,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:00,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:01,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:03,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:06,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:06,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:06,554 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:06,559 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:06,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:07,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:09,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:11,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:11,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:11,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:11,124 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:11,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:13,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:15,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:15,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:15,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:15,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:19,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:19,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:19,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:19,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:19,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:21,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:21,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:23,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:23,160 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:23,160 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:23,165 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:23,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:25,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:26,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:11:27,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:28,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:28,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:28,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:28,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:28,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:30,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:30,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:30,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:31,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:32,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:35,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:35,905 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:35,906 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:35,910 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:35,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:37,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:37,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:40,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:40,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:40,772 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:40,777 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:40,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:43,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:43,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:45,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:45,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:45,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:47,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:47,037 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:47,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:47,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:47,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:49,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:49,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:51,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:51,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:51,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:51,579 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:51,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:53,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:54,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:55,827 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:55,828 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:55,828 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:55,833 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:55,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:56,017 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:11:57,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:00,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:00,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:00,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:00,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:01,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:01,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:01,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:01,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:01,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:03,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:04,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:04,837 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:04,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:04,842 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:04,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:05,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:07,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:09,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:09,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:09,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:09,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:09,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:11,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:11,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:13,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:13,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:13,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:13,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:13,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:15,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:15,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:15,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:15,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:16,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:18,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:18,356 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:18,356 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:18,361 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:18,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:19,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:21,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:22,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:22,296 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:22,296 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:22,301 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:22,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:23,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:26,022 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:12:27,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:27,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:27,055 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:27,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:27,060 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:27,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:29,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:30,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:30,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:30,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:30,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:30,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:30,997 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:31,002 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:31,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:32,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:33,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:35,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:35,423 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:35,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:35,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:35,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:37,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:37,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:39,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:39,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:39,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:39,958 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:39,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:41,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:42,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:43,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:43,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:43,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:43,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:43,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:45,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:45,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:45,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:45,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:48,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:52,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:52,092 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:52,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:52,097 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:52,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:53,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:54,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:56,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:12:57,016 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:57,017 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:57,017 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:57,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:57,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:00,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:00,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:00,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:00,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:00,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:02,908 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:02,909 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:02,909 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:02,914 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:03,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:05,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:06,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:07,548 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:07,549 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:07,549 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:07,554 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:08,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:10,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:11,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:14,430 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:14,431 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:14,431 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:14,436 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:15,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:15,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:15,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:15,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:16,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:17,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:17,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:17,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:17,942 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:17,947 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:18,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:20,021 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:22,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:22,059 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:22,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:22,064 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:23,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:23,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:24,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:26,026 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:13:26,920 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:26,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:26,921 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:26,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:27,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:28,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:30,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:30,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:30,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:32,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:32,537 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:32,537 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:32,542 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:33,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:34,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:34,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:36,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:36,395 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:36,396 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:36,401 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:37,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:38,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:40,260 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:40,261 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:40,261 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:40,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:40,266 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:41,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:42,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:43,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:43,983 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:43,983 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:43,988 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:44,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:45,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:45,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:45,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:45,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:46,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:47,762 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:47,763 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:47,763 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:47,768 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:48,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:50,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:50,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:51,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:51,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:51,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:51,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:52,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:54,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:56,025 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:13:56,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:56,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:56,757 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:56,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:56,762 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:57,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:58,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:00,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:00,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:00,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:00,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:00,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:00,487 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:00,492 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:01,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:01,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:02,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:04,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:04,350 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:04,350 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:04,355 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:05,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:06,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:07,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:09,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:09,177 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:09,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:09,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:10,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:12,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:13,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:13,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:13,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:13,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:13,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:14,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:15,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:15,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:15,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:16,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:17,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:17,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:17,283 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:17,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:18,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:18,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:20,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:22,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:22,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:22,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:22,502 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:23,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:23,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:24,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:26,027 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:14:26,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:26,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:26,234 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:26,239 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:27,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:28,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:29,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:30,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:34,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:34,265 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:34,265 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:34,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:34,270 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:35,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:36,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:37,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:37,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:37,875 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:37,880 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:38,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:39,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:40,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:41,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:41,948 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:41,948 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:41,953 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:42,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:44,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:44,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:45,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:45,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:45,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:45,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:45,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:45,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:45,764 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:46,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:48,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:50,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:50,198 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:50,199 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:50,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:50,204 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:51,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:52,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:53,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:53,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:53,772 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:53,777 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:54,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:55,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:56,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:14:56,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:58,176 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:58,177 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:58,177 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:58,182 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:59,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:00,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:00,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:00,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:00,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:01,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:02,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:02,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:02,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:02,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:03,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:04,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:06,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:06,642 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:06,642 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:06,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:06,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:07,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:08,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:11,648 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:11,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:11,650 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:11,650 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:11,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:12,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:14,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:15,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:15,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:15,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:15,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:15,823 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:15,824 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:15,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:16,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:16,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:18,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:20,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:20,161 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:20,161 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:20,166 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:21,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:22,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:22,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:24,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:24,568 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:24,569 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:24,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:25,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:26,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:15:26,168 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:28,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:28,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:28,988 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:28,988 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:28,993 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:29,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:30,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:32,173 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:32,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:32,988 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:32,988 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:32,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:33,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:33,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:36,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:38,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:40,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:40,573 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:40,574 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:40,578 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:41,180 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:42,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:43,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:45,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:45,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:45,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:49,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:51,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:51,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:51,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:51,109 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:51,189 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:54,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:55,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:56,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:15:56,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:56,054 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:56,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:56,059 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:56,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:58,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:00,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:00,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:00,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:00,416 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:01,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:01,938 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:01,938 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:01,943 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:02,199 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:04,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:05,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:06,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:06,547 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:06,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:06,552 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:07,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:08,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:10,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:10,938 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:10,938 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:10,943 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:11,209 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:11,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:12,212 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:15,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:15,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:15,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:16,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:16,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:16,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:16,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:16,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:17,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:18,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:19,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:19,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:19,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:19,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:20,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:22,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:22,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:23,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:23,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:23,771 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:23,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:24,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:26,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:16:26,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:27,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:27,520 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:27,520 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:27,525 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:28,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:28,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:30,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:30,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:30,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:31,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:31,715 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:31,715 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:31,720 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:32,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:33,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:34,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:36,416 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:36,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:36,417 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:36,422 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:37,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:38,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:39,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:40,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:40,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:40,885 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:40,890 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:41,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:42,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:44,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:45,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:45,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:45,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:47,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:47,523 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:47,523 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:47,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:48,253 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:50,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:50,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:51,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:51,374 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:51,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:51,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:52,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:54,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:56,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:16:56,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:57,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:57,319 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:57,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:57,324 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:58,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:00,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:00,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:00,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:00,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:01,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:05,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:05,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:05,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:05,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:06,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:06,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:08,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:09,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:09,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:09,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:09,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:10,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:12,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:12,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:15,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:15,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:15,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:17,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:17,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:17,926 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:17,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:17,931 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:18,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:20,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:21,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:21,613 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:21,613 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:21,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:22,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:23,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:24,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:26,040 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:17:26,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:26,374 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:26,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:26,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:27,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:28,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:29,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:30,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:30,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:30,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:30,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:30,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:30,432 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:30,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:31,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:32,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:34,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:34,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:34,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:34,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:34,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:35,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:36,299 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:39,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:41,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:41,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:41,679 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:41,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:42,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:44,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:45,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:45,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:45,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:45,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:46,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:46,987 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:46,987 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:46,992 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:47,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:48,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:51,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:51,171 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:51,171 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:51,172 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:51,176 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:51,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:54,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:56,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:17:56,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:57,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:57,751 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:57,751 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:57,756 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:58,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:00,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:00,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:00,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:00,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:01,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:02,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:02,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:02,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:02,320 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:02,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:04,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:06,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:06,371 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:06,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:06,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:06,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:07,328 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:08,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:11,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:11,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:11,535 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:11,535 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:11,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:12,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:14,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:15,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:15,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:15,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:15,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:15,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:15,378 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:15,383 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:16,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:16,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:18,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:19,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:19,391 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:19,391 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:19,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:20,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:21,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:22,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:23,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:23,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:23,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:23,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:24,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:26,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:18:26,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:27,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:27,460 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:27,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:27,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:27,466 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:28,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:30,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:30,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:30,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:30,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:32,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:32,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:32,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:32,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:32,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:33,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:34,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:37,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:37,679 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:37,679 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:37,679 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:37,684 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:38,361 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:40,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:41,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:41,569 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:41,569 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:41,574 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:42,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:43,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:44,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:45,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:45,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:45,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:46,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:46,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:46,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:46,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:47,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:48,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:48,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:52,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:52,892 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:52,892 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:52,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:53,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:54,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:54,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:56,048 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:19:00,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:00,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:00,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:00,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:01,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:01,277 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:01,277 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:01,282 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:01,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:04,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:05,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:05,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:05,814 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:05,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:05,819 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:06,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:08,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:10,819 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:14,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:14,058 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:14,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:14,063 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:14,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:15,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:15,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:15,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:16,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:16,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:18,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:18,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:18,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:18,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:18,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:20,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:21,872 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:21,873 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:21,873 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:21,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:21,878 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:22,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:24,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:25,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:25,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:25,951 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:25,956 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:26,049 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:19:26,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:27,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:28,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:30,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:30,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:30,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:31,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:31,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:31,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:31,692 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:32,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:32,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:34,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:35,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:35,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:35,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:35,729 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:36,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:37,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:38,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:42,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:42,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:42,351 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:42,356 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:42,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:43,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:44,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:45,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:45,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:45,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:46,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:46,635 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:46,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:46,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:47,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:48,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:48,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:50,855 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:50,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:50,856 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:50,861 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:51,431 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:52,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:53,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:54,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:54,930 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:54,930 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:54,935 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:55,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:56,054 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:19:56,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:59,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:59,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:59,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:59,269 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:59,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:00,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:00,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:00,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:00,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:03,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:03,494 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:03,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:03,500 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:04,442 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:04,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:06,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:08,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:08,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:08,239 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:08,244 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:08,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:10,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:10,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:11,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:11,818 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:11,818 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:11,823 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:12,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:14,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:15,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:15,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:15,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:15,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:20,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:20,299 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:20,299 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:20,305 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:20,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:20,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:22,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:24,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:24,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:24,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:24,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:24,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:26,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:20:26,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:26,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:28,232 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:28,233 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:28,233 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:28,238 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:28,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:30,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:30,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:30,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:30,470 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:31,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:33,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:33,730 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:33,730 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:33,735 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:34,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:36,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:36,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:37,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:37,814 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:37,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:37,819 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:38,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:40,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:41,819 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:42,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:42,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:42,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:42,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:42,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:44,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:45,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:45,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:45,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:46,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:46,586 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:46,586 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:46,591 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:47,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:47,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:48,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:50,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:50,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:50,989 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:50,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:51,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:52,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:52,994 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:56,055 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:20:56,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:56,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:56,291 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:56,296 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:56,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:58,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:58,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:00,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:00,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:00,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:02,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:02,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:02,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:02,802 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:03,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:03,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:04,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:08,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:11,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:11,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:11,283 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:11,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:11,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:12,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:14,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:14,895 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:14,895 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:14,896 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:14,901 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:15,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:15,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:15,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:15,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:16,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:19,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:19,102 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:19,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:19,107 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:19,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:20,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:20,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:23,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:23,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:23,240 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:23,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:23,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:24,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:25,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:26,057 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:21:28,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:28,355 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:28,355 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:28,360 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:28,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:30,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:30,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:30,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:30,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:30,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:32,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:32,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:32,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:32,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:32,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:34,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:35,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:40,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:40,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:40,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:40,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:40,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:41,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:42,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:45,022 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:45,022 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:45,023 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:45,027 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:45,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:45,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:45,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:45,539 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:46,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:46,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:48,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:48,992 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:48,992 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:48,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:49,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:50,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:52,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:56,059 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:21:57,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:57,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:57,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:57,408 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:57,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:57,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:58,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:00,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:00,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:00,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:02,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:03,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:03,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:03,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:03,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:03,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:06,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:07,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:08,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:08,345 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:08,345 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:08,351 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:08,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:10,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:13,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:13,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:13,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:13,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:13,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:13,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:14,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:15,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:15,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:15,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:17,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:17,691 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:17,692 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:17,696 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:18,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:18,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:20,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:22,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:22,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:22,542 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:22,547 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:22,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:24,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:24,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:26,064 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:22:30,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:30,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:30,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:30,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:30,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:30,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:30,950 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:30,955 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:31,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:32,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:35,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:35,646 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:35,646 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:35,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:35,651 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:36,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:38,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:40,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:40,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:40,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:40,157 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:40,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:41,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:42,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:45,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:45,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:45,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:45,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:45,414 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:45,414 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:45,419 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:45,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:46,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:46,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:50,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:50,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:50,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:50,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:50,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:52,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:52,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:53,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:53,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:53,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:53,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:54,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:56,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:22:56,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:58,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:58,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:58,913 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:58,913 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:58,919 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:59,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:00,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:00,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:00,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:00,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:02,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:02,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:02,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:02,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:02,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:03,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:04,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:08,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:08,198 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:08,198 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:08,204 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:08,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:09,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:10,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:13,430 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:13,431 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:13,431 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:13,436 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:13,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:14,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:14,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:15,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:15,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:15,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:17,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:17,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:17,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:17,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:18,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:19,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:20,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:23,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:23,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:23,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:23,331 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:23,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:24,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:25,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:26,068 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:23:27,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:27,954 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:27,954 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:27,959 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:28,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:30,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:30,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:30,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:30,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:30,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:32,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:32,643 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:32,643 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:32,648 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:32,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:34,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:35,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:36,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:36,607 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:36,607 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:36,612 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:36,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:38,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:41,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:42,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:42,140 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:42,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:42,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:42,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:44,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:45,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:45,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:45,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:46,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:46,728 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:46,728 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:46,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:46,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:47,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:48,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:51,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:53,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:53,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:53,640 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:53,645 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:53,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:56,067 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:23:56,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:57,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:57,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:57,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:57,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:57,811 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:58,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:00,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:00,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:00,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:00,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:02,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:02,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:02,729 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:02,729 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:02,734 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:03,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:04,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:06,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:06,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:06,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:06,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:06,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:07,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:08,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:10,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:10,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:10,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:10,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:10,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:12,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:13,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:14,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:14,494 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:14,494 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:14,499 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:14,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:15,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:15,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:15,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:16,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:19,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:19,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:19,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:19,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:19,123 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:19,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:20,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:22,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:22,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:22,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:22,795 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:23,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:24,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:24,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:26,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:24:27,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:27,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:27,214 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:27,219 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:27,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:28,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:30,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:30,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:32,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:32,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:32,919 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:32,924 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:33,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:34,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:35,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:39,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:39,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:39,686 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:39,691 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:39,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:41,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:42,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:44,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:44,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:44,373 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:44,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:44,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:45,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:45,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:45,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:46,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:47,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:48,700 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:48,700 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:48,701 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:48,705 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:48,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:50,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:52,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:54,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:54,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:54,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:54,391 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:54,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:56,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:24:56,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:58,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:58,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:58,534 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:58,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:58,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:58,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:00,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:00,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:00,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:00,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:03,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:08,192 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:08,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:08,192 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:08,197 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:08,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:09,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:10,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:11,520 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:11,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:11,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:11,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:11,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:12,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:14,526 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:15,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:15,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:15,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:16,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:16,440 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:16,440 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:16,444 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:16,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:18,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:20,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:20,391 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:20,391 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:20,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:20,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:20,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:22,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:24,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:24,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:24,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:24,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:24,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:25,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:26,076 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:25:26,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:28,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:28,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:28,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:28,121 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:28,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:30,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:30,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:31,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:32,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:32,464 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:32,464 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:32,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:32,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:34,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:36,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:36,475 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:36,475 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:36,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:36,480 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:36,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:38,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:41,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:42,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:42,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:42,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:42,610 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:42,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:44,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:45,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:45,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:45,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:46,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:46,405 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:46,405 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:46,410 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:46,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:47,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:48,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:50,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:50,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:50,366 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:50,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:50,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:52,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:53,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:54,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:54,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:54,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:54,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:54,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:56,076 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:25:56,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:58,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:58,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:58,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:58,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:58,407 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:58,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:00,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:00,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:00,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:00,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:02,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:02,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:02,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:02,374 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:02,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:03,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:04,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:06,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:06,506 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:06,507 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:06,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:06,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:08,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:08,809 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:10,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:10,504 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:10,504 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:10,509 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:10,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:12,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:14,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:14,907 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:14,908 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:14,908 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:14,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:15,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:15,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:15,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:15,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:16,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:18,871 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:18,872 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:18,872 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:18,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:19,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:19,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:20,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:23,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:23,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:23,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:23,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:23,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:24,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:25,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:26,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:26:29,831 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:29,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:29,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:29,836 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:30,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:30,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:31,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:32,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:34,803 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:34,804 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:34,804 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:34,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:34,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:36,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:36,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:38,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:38,771 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:38,772 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:38,777 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:38,839 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:40,842 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:42,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:45,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:45,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:45,416 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:45,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:45,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:45,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:45,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:46,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:48,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:51,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:51,248 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:51,248 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:51,253 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:51,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:52,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:54,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:55,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:55,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:55,029 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:55,034 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:55,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:56,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:26:56,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:59,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:59,560 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:59,560 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:59,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:59,565 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:59,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:00,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:00,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:00,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:00,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:03,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:03,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:03,248 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:03,252 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:03,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:04,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:05,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:08,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:08,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:08,209 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:08,214 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:08,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:10,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:11,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:12,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:12,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:12,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:12,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:12,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:14,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:15,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:15,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:15,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:16,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:16,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:16,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:16,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:16,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:17,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:18,880 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:22,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:24,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:24,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:24,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:24,557 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:24,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:26,082 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:27:26,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:28,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:29,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:29,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:29,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:29,373 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:29,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:30,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:30,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:30,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:33,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:33,935 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:33,936 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:33,936 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:33,941 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:34,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:36,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:38,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:39,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:39,673 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:39,673 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:39,677 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:39,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:40,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:44,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:45,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:45,210 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:45,384 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:45,384 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:45,388 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:45,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:46,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:49,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:49,055 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:49,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:49,060 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:49,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:50,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:50,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:55,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:55,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:55,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:55,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:55,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:56,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:56,084 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:27:56,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:00,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:00,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:00,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:01,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:01,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:01,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:01,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:01,210 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:01,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:02,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:05,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:05,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:05,947 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:05,951 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:06,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:06,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:08,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:11,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:11,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:11,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:11,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:11,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:12,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:12,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:15,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:15,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:15,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:15,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:15,714 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:15,714 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:15,719 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:15,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:16,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:17,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:21,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:21,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:21,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:21,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:21,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:22,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:22,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:25,875 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:25,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:25,876 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:25,881 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:25,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:26,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:28:28,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:28,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:29,452 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:29,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:29,453 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:29,458 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:29,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:30,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:30,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:30,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:30,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:33,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:35,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:35,858 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:35,858 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:35,864 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:35,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:38,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:38,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:39,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:39,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:39,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:39,444 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:39,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:40,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:43,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:43,979 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:43,980 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:43,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:43,984 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:44,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:45,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:46,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:47,839 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:47,840 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:47,840 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:47,845 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:47,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:49,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:50,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:52,900 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:52,900 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:52,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:52,905 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:52,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:54,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:54,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:56,091 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:28:57,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:57,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:57,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:57,610 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:57,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:58,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:00,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:00,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:00,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:00,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:05,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:06,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:06,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:06,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:06,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:06,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:08,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:11,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:12,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:12,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:12,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:12,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:12,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:14,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:15,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:15,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:15,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:17,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:17,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:17,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:17,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:17,550 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:17,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:18,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:21,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:21,611 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:21,611 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:21,616 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:21,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:22,616 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:22,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:26,090 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:29:26,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:26,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:26,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:26,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:26,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:28,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:28,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:30,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:30,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:30,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:34,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:35,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:35,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:35,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:35,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:36,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:37,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:39,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:39,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:39,860 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:39,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:39,865 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:40,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:41,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:44,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:44,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:44,542 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:44,547 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:45,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:45,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:45,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:45,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:47,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:49,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:49,485 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:49,485 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:49,490 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:50,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:50,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:51,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:55,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:56,098 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:29:58,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:58,601 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:58,601 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:58,606 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:59,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:00,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:00,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:00,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:01,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:01,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:04,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:04,933 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:04,934 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:04,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:05,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:06,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:07,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:09,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:09,429 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:09,430 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:09,434 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:10,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:11,037 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:12,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:14,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:14,314 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:14,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:14,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:15,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:15,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:15,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:15,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:17,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:18,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:19,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:19,606 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:19,606 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:19,612 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:20,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:21,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:23,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:23,479 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:23,480 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:23,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:23,484 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:24,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:25,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:26,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:30:27,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:27,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:27,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:27,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:28,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:28,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:29,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:30,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:30,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:30,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:31,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:31,500 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:31,500 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:31,505 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:32,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:33,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:33,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:35,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:35,690 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:35,690 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:35,695 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:36,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:37,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:38,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:39,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:39,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:39,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:39,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:40,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:41,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:43,416 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:43,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:43,417 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:43,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:44,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:44,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:45,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:45,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:45,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:45,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:47,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:47,220 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:47,220 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:47,225 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:48,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:49,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:50,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:50,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:50,960 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:50,960 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:50,966 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:51,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:53,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:54,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:54,595 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:54,595 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:54,600 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:55,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:55,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:56,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:30:57,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:59,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:59,396 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:59,396 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:59,401 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:00,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:00,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:00,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:00,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:01,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:01,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:05,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:05,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:05,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:05,041 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:05,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:07,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:07,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:09,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:09,221 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:09,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:09,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:10,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:11,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:12,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:13,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:13,411 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:13,412 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:13,416 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:14,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:15,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:15,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:15,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:15,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:17,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:17,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:17,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:17,950 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:17,954 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:18,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:19,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:22,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:23,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:23,847 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:23,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:23,855 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:24,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:25,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:26,097 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:31:28,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:29,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:29,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:29,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:29,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:30,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:30,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:30,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:30,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:31,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:33,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:33,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:33,922 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:33,922 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:33,927 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:34,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:35,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:38,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:38,459 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:38,459 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:38,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:38,464 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:39,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:41,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:43,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:45,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:45,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:45,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:47,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:47,048 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:47,048 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:47,053 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:47,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:49,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:49,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:53,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:53,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:53,347 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:53,352 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:54,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:54,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:55,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:56,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:31:56,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:56,983 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:56,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:56,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:57,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:59,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:59,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:00,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:00,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:00,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:03,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:03,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:03,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:03,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:04,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:05,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:05,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:08,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:08,107 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:08,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:08,112 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:08,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:11,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:11,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:12,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:12,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:12,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:12,113 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:12,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:15,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:15,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:15,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:15,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:16,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:21,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:21,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:21,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:21,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:21,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:22,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:23,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:26,105 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:32:26,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:26,422 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:26,422 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:26,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:26,427 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:27,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:29,209 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:30,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:30,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:30,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:31,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:31,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:31,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:31,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:31,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:32,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:33,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:36,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:36,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:36,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:36,998 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:37,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:37,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:39,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:41,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:41,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:41,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:41,292 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:42,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:43,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:43,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:45,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:45,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:45,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:45,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:45,419 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:45,419 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:45,424 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:46,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:47,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:48,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:51,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:51,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:51,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:51,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:52,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:53,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:53,468 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:56,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:32:56,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:56,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:56,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:56,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:57,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:59,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:00,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:00,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:00,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:00,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:00,419 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:00,419 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:00,424 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:01,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:03,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:04,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:04,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:04,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:04,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:04,579 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:05,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:07,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:08,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:08,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:08,956 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:08,961 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:09,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:09,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:11,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:14,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:14,162 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:14,162 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:14,167 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:14,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:15,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:15,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:15,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:15,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:15,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:19,900 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:19,901 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:19,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:19,906 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:20,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:20,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:21,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:23,616 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:23,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:23,617 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:23,622 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:24,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:25,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:26,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:33:26,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:28,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:28,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:28,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:28,277 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:28,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:30,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:30,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:30,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:31,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:31,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:32,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:32,971 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:32,972 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:32,976 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:33,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:35,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:36,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:36,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:36,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:36,806 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:36,811 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:37,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:39,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:40,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:40,750 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:40,751 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:40,755 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:41,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:42,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:43,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:44,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:44,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:44,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:44,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:45,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:45,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:45,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:45,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:47,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:48,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:48,759 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:48,759 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:48,759 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:48,764 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:49,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:51,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:52,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:52,589 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:52,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:52,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:53,321 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:53,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:55,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:56,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:33:57,976 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:57,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:57,977 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:57,981 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:58,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:58,982 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:59,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:00,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:00,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:04,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:06,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:06,412 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:06,412 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:06,417 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:07,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:09,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:09,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:11,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:11,560 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:11,560 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:11,565 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:12,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:13,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:14,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:15,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:15,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:15,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:15,856 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:15,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:15,856 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:15,861 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:16,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:17,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:19,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:21,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:21,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:21,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:21,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:22,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:23,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:25,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:25,998 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:25,999 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:25,999 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:26,004 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:26,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:34:26,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:27,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:30,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:30,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:30,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:31,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:31,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:31,704 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:31,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:31,709 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:32,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:33,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:36,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:37,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:37,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:37,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:37,320 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:37,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:39,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:42,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:42,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:42,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:42,624 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:42,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:43,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:45,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:45,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:45,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:45,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:46,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:46,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:46,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:46,630 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:47,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:47,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:49,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:50,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:50,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:50,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:50,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:51,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:53,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:53,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:55,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:55,889 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:55,889 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:55,894 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:56,112 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:34:56,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:57,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:59,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:00,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:00,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:01,530 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:01,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:01,531 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:01,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:02,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:03,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:04,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:07,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:07,672 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:07,672 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:07,677 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:08,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:09,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:09,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:12,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:12,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:12,210 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:12,215 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:12,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:13,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:15,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:15,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:15,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:15,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:16,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:16,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:16,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:16,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:16,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:17,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:19,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:19,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:19,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:19,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:20,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:20,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:21,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:24,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:24,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:24,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:24,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:24,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:26,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:35:26,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:27,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:28,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:28,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:28,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:28,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:29,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:30,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:30,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:30,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:31,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:31,409 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:32,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:32,452 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:32,452 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:32,457 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:33,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:35,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:36,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:37,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:37,480 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:37,480 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:37,485 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:38,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:39,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:41,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:41,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:41,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:41,094 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:41,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:42,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:43,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:44,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:44,847 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:44,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:44,853 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:45,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:45,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:45,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:45,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:47,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:47,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:48,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:48,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:48,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:48,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:49,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:51,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:52,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:55,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:55,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:55,148 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:55,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:55,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:56,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:35:57,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:58,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:59,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:59,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:59,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:59,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:00,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:00,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:01,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:03,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:03,679 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:03,680 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:03,680 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:03,685 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:04,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:05,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:07,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:07,491 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:07,492 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:07,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:08,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:08,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:09,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:12,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:12,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:12,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:12,490 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:13,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:14,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:15,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:15,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:15,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:15,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:16,986 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:16,987 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:16,987 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:16,993 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:17,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:19,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:19,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:21,156 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:21,157 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:21,157 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:21,162 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:21,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:23,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:25,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:26,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:36:27,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:27,076 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:27,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:27,081 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:27,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:29,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:30,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:30,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:30,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:30,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:30,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:30,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:30,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:30,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:31,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:33,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:35,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:36,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:36,878 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:36,878 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:36,883 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:37,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:39,477 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:40,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:40,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:40,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:40,899 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:40,904 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:41,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:43,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:44,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:44,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:44,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:45,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:45,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:45,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:46,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:47,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:51,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:53,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:53,344 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:53,344 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:53,349 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:53,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:55,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:56,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:36:57,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:58,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:58,019 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:58,019 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:58,024 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:58,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:59,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:00,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:02,416 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:03,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:03,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:03,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:03,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:03,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:05,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:08,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:08,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:08,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:08,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:08,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:08,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:11,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:13,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:13,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:13,147 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:13,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:13,152 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:13,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:15,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:15,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:15,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:17,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:17,557 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:17,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:17,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:18,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:18,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:19,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:22,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:22,716 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:22,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:22,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:23,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:23,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:25,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:26,122 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:37:27,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:27,134 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:27,135 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:27,140 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:27,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:29,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:29,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:30,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:30,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:30,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:31,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:31,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:31,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:31,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:32,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:33,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:34,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:36,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:36,341 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:36,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:36,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:36,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:37,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:40,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:41,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:41,576 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:41,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:41,582 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:42,536 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:43,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:45,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:45,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:46,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:46,240 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:46,240 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:46,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:46,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:47,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:50,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:50,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:50,030 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:50,035 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:50,546 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:51,035 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:51,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:54,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:54,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:54,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:54,915 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:55,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:56,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:37:56,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:57,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:59,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:59,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:59,556 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:59,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:00,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:00,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:00,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:01,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:01,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:04,012 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:04,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:04,013 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:04,018 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:04,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:05,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:07,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:07,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:07,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:07,778 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:07,783 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:08,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:09,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:12,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:12,041 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:12,041 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:12,047 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:12,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:13,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:13,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:15,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:16,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:16,611 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:16,611 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:16,616 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:17,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:18,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:19,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:19,966 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:19,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:19,967 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:19,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:20,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:21,578 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:23,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:26,126 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:38:28,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:28,220 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:28,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:28,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:28,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:29,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:29,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:30,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:30,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:30,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:32,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:32,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:32,004 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:32,009 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:32,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:33,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:35,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:37,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:37,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:37,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:37,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:37,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:39,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:41,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:41,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:41,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:41,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:41,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:41,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:43,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:45,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:46,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:46,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:46,721 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:46,722 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:46,726 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:47,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:49,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:51,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:54,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:54,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:54,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:54,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:55,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:56,131 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:38:57,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:57,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:58,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:58,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:58,990 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:58,995 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:59,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:00,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:00,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:00,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:01,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:02,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:02,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:02,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:02,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:02,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:03,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:05,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:06,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:06,918 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:06,919 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:06,923 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:07,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:07,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:09,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:10,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:10,664 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:10,664 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:10,669 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:11,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:13,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:13,670 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:14,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:14,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:14,824 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:14,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:15,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:15,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:17,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:18,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:18,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:18,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:18,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:18,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:19,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:21,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:22,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:22,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:22,349 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:22,354 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:22,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:23,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:25,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:26,130 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:39:26,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:26,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:26,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:26,328 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:26,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:27,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:30,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:30,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:30,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:30,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:30,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:30,712 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:30,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:30,717 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:31,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:33,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:35,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:36,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:36,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:36,176 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:36,181 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:36,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:37,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:41,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:41,482 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:41,483 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:41,483 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:41,488 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:41,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:43,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:45,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:45,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:45,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:45,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:45,660 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:45,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:46,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:47,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:49,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:49,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:49,951 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:49,956 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:50,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:51,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:51,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:54,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:54,164 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:54,164 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:54,169 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:54,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:55,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:56,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:39:57,136 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:00,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:00,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:00,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:00,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:00,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:00,624 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:00,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:00,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:02,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:03,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:04,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:04,376 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:04,376 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:04,381 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:04,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:05,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:08,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:08,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:08,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:08,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:08,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:09,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:11,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:13,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:13,147 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:13,147 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:13,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:13,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:14,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:15,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:15,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:15,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:17,736 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:17,737 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:17,737 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:17,742 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:18,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:19,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:19,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:23,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:23,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:23,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:23,190 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:23,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:25,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:25,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:26,134 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:40:28,259 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:28,260 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:28,260 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:28,265 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:28,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:29,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:30,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:30,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:30,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:30,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:35,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:36,748 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:36,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:36,749 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:36,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:37,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:39,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:40,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:41,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:41,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:41,854 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:41,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:42,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:43,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:45,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:45,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:45,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:46,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:47,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:47,750 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:47,750 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:47,756 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:48,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:49,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:51,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:51,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:51,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:51,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:51,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:51,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:53,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:55,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:55,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:55,413 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:55,418 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:55,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:56,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:40:57,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:57,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:00,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:00,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:00,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:00,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:00,325 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:00,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:00,331 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:00,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:01,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:02,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:07,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:08,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:08,693 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:08,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:08,698 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:08,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:11,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:12,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:15,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:15,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:15,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:15,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:15,322 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:15,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:15,327 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:15,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:17,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:18,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:19,139 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:19,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:19,140 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:19,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:19,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:21,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:23,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:23,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:23,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:23,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:23,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:24,230 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:25,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:26,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:41:27,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:27,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:27,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:27,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:27,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:29,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:29,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:30,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:30,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:30,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:33,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:33,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:33,363 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:33,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:33,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:35,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:35,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:37,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:37,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:37,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:37,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:37,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:39,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:40,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:41,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:41,171 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:41,171 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:41,176 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:41,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:43,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:45,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:45,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:45,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:46,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:51,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:54,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:54,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:54,378 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:54,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:54,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:55,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:56,140 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:41:57,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:59,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:59,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:59,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:59,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:59,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:00,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:00,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:00,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:01,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:02,392 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:03,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:03,418 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:03,418 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:03,422 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:03,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:05,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:07,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:07,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:07,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:07,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:07,697 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:07,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:09,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:12,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:13,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:13,160 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:13,160 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:13,164 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:13,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:15,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:15,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:15,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:15,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:16,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:16,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:16,927 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:16,931 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:17,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:17,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:19,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:20,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:20,417 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:20,418 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:20,422 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:20,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:21,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:23,423 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:26,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:42:26,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:26,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:26,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:26,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:27,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:28,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:29,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:30,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:30,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:30,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:31,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:31,296 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:31,296 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:31,301 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:31,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:33,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:34,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:35,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:35,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:35,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:35,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:35,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:37,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:39,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:41,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:41,419 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:41,419 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:41,424 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:41,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:43,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:45,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:45,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:45,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:45,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:46,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:46,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:46,536 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:46,541 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:46,842 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:47,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:50,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:51,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:51,244 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:51,244 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:51,249 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:51,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:53,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:55,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:55,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:55,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:55,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:55,853 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:55,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:56,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:42:57,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:59,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:00,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:00,000 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:00,005 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:00,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:00,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:00,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:00,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:01,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:01,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:06,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:09,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:09,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:09,282 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:09,287 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:09,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:11,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:12,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:13,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:13,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:13,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:13,314 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:13,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:15,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:15,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:15,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:15,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:17,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:17,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:17,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:17,541 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:17,546 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:17,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:19,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:21,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:21,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:21,899 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:21,904 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:22,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:22,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:23,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:25,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:25,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:25,776 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:25,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:25,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:26,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:43:27,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:28,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:29,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:29,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:29,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:29,746 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:29,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:30,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:30,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:30,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:31,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:33,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:33,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:33,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:33,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:33,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:34,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:35,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:38,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:38,126 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:38,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:38,131 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:38,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:39,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:39,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:44,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:44,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:44,747 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:44,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:44,752 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:44,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:45,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:45,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:45,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:45,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:49,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:50,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:50,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:50,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:50,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:50,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:51,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:55,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:56,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:56,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:56,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:56,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:56,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:43:56,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:57,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:59,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:59,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:59,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:59,795 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:59,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:00,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:00,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:00,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:00,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:01,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:05,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:05,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:05,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:05,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:05,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:05,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:07,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:10,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:10,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:10,642 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:10,642 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:10,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:10,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:11,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:14,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:14,823 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:14,823 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:14,828 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:14,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:15,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:15,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:15,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:15,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:16,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:19,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:19,143 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:19,144 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:19,148 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:19,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:21,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:22,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:26,153 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:44:27,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:27,565 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:27,565 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:27,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:27,569 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:27,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:29,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:30,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:30,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:30,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:33,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:36,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:36,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:36,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:36,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:36,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:37,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:39,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:42,258 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:42,259 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:42,260 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:42,264 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:42,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:43,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:44,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:45,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:45,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:45,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:47,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:47,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:47,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:47,057 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:47,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:49,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:50,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:51,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:51,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:51,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:51,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:51,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:53,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:55,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:55,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:55,550 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:55,550 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:55,555 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:55,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:56,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:44:57,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:59,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:59,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:59,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:59,750 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:59,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:00,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:00,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:00,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:01,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:04,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:04,276 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:04,277 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:04,281 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:04,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:05,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:06,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:08,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:08,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:08,956 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:08,961 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:08,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:09,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:11,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:15,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:15,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:15,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:16,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:16,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:16,009 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:16,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:16,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:17,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:17,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:20,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:20,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:20,634 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:20,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:20,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:21,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:22,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:24,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:24,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:24,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:24,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:24,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:25,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:26,157 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:45:28,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:28,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:28,212 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:28,212 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:28,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:28,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:30,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:30,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:30,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:30,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:33,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:36,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:36,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:36,492 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:36,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:37,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:38,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:38,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:41,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:41,049 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:41,050 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:41,055 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:42,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:44,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:45,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:45,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:45,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:49,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:49,296 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:49,296 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:49,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:49,301 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:50,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:52,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:53,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:53,226 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:53,226 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:53,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:54,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:55,232 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:56,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:56,156 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:45:58,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:58,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:58,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:58,106 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:59,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:00,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:00,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:00,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:00,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:02,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:02,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:02,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:02,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:03,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:04,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:05,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:06,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:06,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:06,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:06,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:07,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:08,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:10,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:10,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:10,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:10,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:11,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:11,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:12,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:14,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:14,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:14,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:14,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:15,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:15,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:15,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:15,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:16,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:17,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:19,664 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:19,665 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:19,666 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:19,670 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:20,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:22,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:22,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:23,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:23,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:23,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:23,853 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:24,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:26,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:26,161 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:46:28,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:29,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:29,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:29,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:29,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:30,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:30,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:30,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:30,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:32,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:33,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:33,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:33,589 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:33,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:33,594 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:34,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:36,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:38,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:42,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:42,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:42,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:42,119 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:43,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:44,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:44,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:45,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:45,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:45,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:46,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:46,968 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:46,969 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:46,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:47,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:48,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:49,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:50,811 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:50,811 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:50,812 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:50,817 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:51,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:52,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:54,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:54,798 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:54,798 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:54,803 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:55,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:55,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:56,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:56,160 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:46:58,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:58,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:58,731 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:58,736 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:59,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:00,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:00,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:00,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:01,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:02,426 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:02,427 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:02,427 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:02,433 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:03,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:04,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:06,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:10,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:10,468 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:10,468 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:10,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:11,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:11,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:12,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:14,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:14,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:14,217 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:14,222 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:15,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:15,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:15,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:15,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:16,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:17,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:21,398 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:21,399 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:21,400 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:21,405 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:22,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:22,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:24,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:26,162 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:47:26,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:26,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:26,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:26,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:27,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:28,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:28,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:30,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:30,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:30,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:30,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:30,381 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:30,382 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:30,387 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:31,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:32,116 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:33,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:33,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:33,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:33,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:33,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:34,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:36,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:37,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:37,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:37,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:37,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:38,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:38,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:40,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:42,194 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:42,195 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:42,195 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:42,200 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:43,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:44,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:44,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:45,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:45,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:45,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:46,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:46,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:46,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:46,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:47,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:48,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:49,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:51,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:51,269 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:51,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:51,274 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:52,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:54,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:55,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:55,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:55,198 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:55,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:55,203 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:56,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:56,164 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:47:58,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:59,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:59,901 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:59,902 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:59,907 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:00,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:00,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:00,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:00,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:02,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:04,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:04,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:04,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:04,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:04,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:06,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:06,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:07,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:07,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:07,734 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:07,739 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:08,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:10,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:11,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:13,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:13,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:13,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:13,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:13,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:14,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:15,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:15,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:15,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:17,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:21,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:21,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:21,664 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:21,669 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:22,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:22,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:24,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:26,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:48:26,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:26,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:26,692 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:26,697 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:27,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:27,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:28,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:30,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:30,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:30,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:32,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:32,070 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:32,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:32,075 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:32,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:33,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:34,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:36,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:36,412 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:36,412 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:36,418 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:37,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:38,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:38,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:41,960 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:41,961 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:41,961 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:41,966 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:42,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:43,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:44,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:45,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:45,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:45,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:48,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:48,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:48,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:48,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:48,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:49,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:50,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:54,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:56,168 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:48:56,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:56,219 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:56,219 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:56,225 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:57,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:58,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:59,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:00,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:00,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:01,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:01,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:01,666 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:01,672 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:02,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:04,201 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:04,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:05,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:05,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:05,913 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:05,918 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:06,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:08,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:09,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:12,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:12,196 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:12,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:12,201 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:12,209 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:14,212 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:15,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:15,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:15,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:15,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:17,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:17,024 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:17,024 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:17,029 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:17,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:18,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:21,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:21,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:21,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:21,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:21,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:22,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:22,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:25,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:25,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:25,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:25,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:26,171 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:49:26,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:26,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:28,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:30,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:30,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:30,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:31,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:33,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:33,212 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:33,212 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:33,217 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:33,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:34,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:37,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:37,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:37,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:37,506 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:37,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:38,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:40,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:42,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:42,168 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:42,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:42,173 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:42,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:43,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:44,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:45,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:45,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:45,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:46,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:46,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:46,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:46,337 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:47,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:48,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:48,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:49,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:49,900 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:49,900 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:49,905 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:50,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:52,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:53,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:56,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:49:58,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:58,412 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:58,412 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:58,418 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:59,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:59,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:00,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:00,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:00,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:02,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:02,960 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:02,960 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:02,966 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:03,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:04,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:04,966 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:08,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:08,418 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:08,418 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:08,424 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:09,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:10,263 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:10,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:12,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:12,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:12,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:12,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:13,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:14,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:15,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:15,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:15,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:16,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:17,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:17,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:17,303 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:17,308 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:18,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:18,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:21,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:21,301 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:21,301 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:21,306 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:22,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:22,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:22,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:26,175 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:50:27,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:27,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:27,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:27,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:27,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:28,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:28,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:30,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:30,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:30,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:30,824 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:30,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:30,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:30,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:31,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:32,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:33,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:34,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:34,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:34,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:34,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:35,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:36,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:39,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:43,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:43,313 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:43,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:43,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:44,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:44,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:45,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:45,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:45,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:45,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:48,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:48,127 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:48,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:48,133 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:48,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:50,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:50,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:53,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:53,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:53,207 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:53,212 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:53,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:54,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:55,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:56,180 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:51:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:00,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:00,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:00,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:02,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:02,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:02,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:02,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:03,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:04,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:05,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:07,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:07,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:07,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:07,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:07,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:08,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:10,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:10,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:10,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:10,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:10,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:11,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:12,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:15,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:15,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:15,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:15,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:15,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:15,349 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:15,355 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:16,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:16,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:16,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:21,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:21,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:21,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:21,505 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:21,510 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:22,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:24,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:26,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:26,134 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:26,135 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:26,140 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:26,179 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:51:26,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:27,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:28,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:29,958 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:29,959 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:29,959 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:29,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:30,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:30,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:30,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:30,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:32,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:32,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:33,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:33,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:33,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:33,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:34,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:36,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:37,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:37,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:37,420 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:37,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:37,425 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:38,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:40,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:41,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:41,352 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:41,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:41,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:42,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:42,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:43,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:45,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:45,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:45,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:47,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:47,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:47,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:47,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:48,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:48,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:50,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:50,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:50,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:50,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:50,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:51,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:52,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:53,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:56,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:56,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:56,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:56,043 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:56,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:51:56,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:58,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:59,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:00,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:00,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:00,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:04,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:04,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:04,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:04,544 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:04,549 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:05,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:06,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:09,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:09,727 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:09,728 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:09,728 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:09,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:10,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:12,375 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:13,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:13,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:13,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:13,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:13,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:14,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:15,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:15,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:15,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:15,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:19,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:19,477 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:19,477 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:19,484 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:20,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:20,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:22,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:23,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:23,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:23,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:23,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:24,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:25,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:26,183 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:52:26,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:30,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:30,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:30,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:30,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:30,897 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:30,898 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:30,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:30,903 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:31,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:32,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:34,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:34,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:34,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:34,802 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:35,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:36,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:36,803 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:38,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:38,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:38,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:38,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:39,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:40,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:42,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:42,418 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:42,418 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:42,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:42,423 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:43,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:44,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:45,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:45,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:45,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:48,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:49,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:49,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:49,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:49,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:50,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:52,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:53,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:53,806 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:53,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:53,807 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:53,812 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:54,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:56,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:52:56,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:58,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:58,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:58,222 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:58,227 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:58,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:59,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:00,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:00,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:00,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:00,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:04,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:04,154 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:04,155 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:04,159 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:04,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:05,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:06,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:09,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:09,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:09,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:09,056 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:09,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:10,431 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:11,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:13,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:13,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:13,464 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:13,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:14,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:14,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:15,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:15,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:15,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:16,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:18,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:18,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:18,347 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:18,352 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:18,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:20,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:22,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:23,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:23,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:23,035 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:23,039 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:23,442 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:24,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:26,187 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:53:28,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:28,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:28,596 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:28,596 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:28,601 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:29,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:30,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:30,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:30,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:30,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:32,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:32,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:32,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:32,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:33,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:33,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:34,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:36,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:36,308 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:36,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:36,314 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:36,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:38,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:39,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:39,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:39,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:39,727 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:39,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:40,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:42,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:44,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:45,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:45,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:45,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:45,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:45,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:45,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:45,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:46,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:48,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:49,752 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:49,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:49,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:49,923 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:49,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:50,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:52,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:54,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:54,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:54,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:54,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:54,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:55,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:56,189 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:53:56,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:59,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:59,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:59,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:59,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:00,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:00,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:00,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:02,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:03,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:03,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:03,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:03,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:03,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:04,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:05,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:07,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:07,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:07,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:07,886 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:08,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:10,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:10,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:13,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:13,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:13,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:13,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:13,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:14,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:15,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:15,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:15,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:16,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:16,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:16,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:16,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:16,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:17,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:18,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:21,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:21,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:21,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:21,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:21,468 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:21,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:22,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:26,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:54:26,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:26,756 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:26,757 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:26,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:26,762 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:27,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:28,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:30,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:30,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:30,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:30,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:30,892 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:30,892 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:30,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:31,511 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:31,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:32,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:35,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:35,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:35,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:35,325 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:35,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:36,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:37,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:39,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:39,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:39,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:39,328 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:39,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:40,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:42,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:42,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:42,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:42,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:42,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:43,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:44,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:45,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:45,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:45,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:47,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:47,955 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:47,956 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:47,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:47,961 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:48,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:50,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:52,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:52,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:52,240 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:52,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:52,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:53,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:54,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:55,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:55,643 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:55,643 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:55,648 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:56,193 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:54:56,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:56,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:59,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:59,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:59,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:59,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:59,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:00,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:00,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:00,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:02,546 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:04,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:04,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:04,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:04,477 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:04,482 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:04,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:06,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:09,482 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:10,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:10,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:10,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:10,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:11,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:12,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:14,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:14,221 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:14,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:14,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:14,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:15,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:15,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:15,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:16,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:19,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:19,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:19,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:19,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:19,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:20,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:21,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:25,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:25,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:25,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:25,152 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:25,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:26,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:26,195 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:55:26,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:29,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:29,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:29,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:29,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:30,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:30,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:30,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:30,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:31,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:32,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:34,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:34,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:34,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:34,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:35,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:36,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:36,628 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:39,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:39,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:39,009 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:39,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:39,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:40,580 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:42,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:43,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:43,024 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:43,024 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:43,029 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:43,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:44,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:45,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:45,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:47,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:47,301 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:47,302 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:47,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:47,307 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:47,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:48,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:52,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:53,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:53,126 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:53,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:53,131 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:53,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:54,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:56,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:55:56,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:56,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:56,662 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:56,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:57,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:57,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:58,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:00,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:00,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:03,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:05,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:05,635 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:05,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:05,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:06,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:06,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:08,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:10,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:10,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:10,662 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:10,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:11,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:12,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:13,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:14,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:14,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:14,750 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:14,755 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:15,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:15,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:15,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:16,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:18,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:18,441 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:18,442 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:18,446 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:18,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:19,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:20,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:24,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:26,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:26,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:26,142 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:26,147 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:26,202 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:56:26,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:28,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:30,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:30,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:30,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:30,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:30,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:30,814 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:31,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:32,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:34,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:34,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:34,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:34,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:35,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:35,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:36,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:38,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:38,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:38,622 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:38,627 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:38,635 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:40,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:41,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:43,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:43,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:43,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:43,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:43,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:44,642 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:45,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:45,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:47,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:48,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:48,466 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:48,466 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:48,471 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:48,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:50,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:52,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:52,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:52,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:52,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:52,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:53,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:54,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:56,201 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:56:57,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:57,157 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:57,158 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:57,163 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:57,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:58,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:58,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:00,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:00,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:00,871 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:00,871 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:00,872 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:00,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:01,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:02,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:03,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:04,911 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:04,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:04,912 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:04,917 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:05,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:06,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:08,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:10,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:10,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:10,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:10,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:10,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:12,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:14,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:14,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:14,614 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:14,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:14,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:14,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:15,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:15,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:16,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:19,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:19,882 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:19,882 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:19,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:19,887 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:20,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:22,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:24,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:25,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:25,015 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:25,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:25,020 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:25,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:26,206 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:57:26,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:29,931 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:29,932 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:29,933 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:29,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:29,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:30,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:30,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:30,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:32,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:33,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:33,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:33,938 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:33,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:34,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:34,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:36,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:38,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:38,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:38,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:38,117 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:38,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:40,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:40,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:43,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:43,082 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:43,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:43,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:43,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:44,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:45,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:45,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:45,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:49,232 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:49,233 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:49,233 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:49,238 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:49,708 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:50,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:51,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:55,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:55,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:55,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:55,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:55,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:56,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:57:56,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:57,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:59,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:59,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:59,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:59,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:59,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:00,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:00,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:00,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:02,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:05,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:05,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:05,555 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:05,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:05,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:06,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:07,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:12,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:12,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:12,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:12,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:12,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:13,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:14,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:15,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:15,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:17,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:17,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:17,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:17,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:17,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:18,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:18,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:21,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:21,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:21,421 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:21,426 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:21,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:22,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:23,427 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:24,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:24,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:24,906 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:24,911 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:25,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:26,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:58:26,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:29,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:29,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:29,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:29,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:29,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:29,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:30,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:30,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:30,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:34,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:36,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:36,077 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:36,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:36,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:36,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:38,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:40,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:40,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:40,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:40,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:40,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:40,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:42,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:44,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:44,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:44,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:44,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:44,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:45,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:45,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:45,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:46,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:49,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:49,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:49,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:49,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:49,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:50,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:50,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:54,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:54,269 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:54,270 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:54,275 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:54,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:55,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:56,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:58:56,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:58,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:58,802 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:58,802 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:58,807 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:59,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:00,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:00,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:00,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:00,779 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:02,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:02,940 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:02,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:02,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:03,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:04,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:05,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:07,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:07,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:07,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:07,338 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:07,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:08,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:11,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:11,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:11,577 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:11,577 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:11,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:11,790 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:12,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:15,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:15,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:15,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:15,496 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:15,502 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:15,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:16,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:16,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:21,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:21,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:21,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:21,321 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:21,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:22,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:22,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:25,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:25,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:25,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:25,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:25,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:26,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:59:26,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:28,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:30,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:30,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:30,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:30,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:30,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:30,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:30,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:32,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:33,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:34,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:34,982 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:34,982 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:34,987 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:35,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:36,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:38,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:39,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:39,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:39,210 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:39,215 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:39,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:40,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:44,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:44,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:44,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:44,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:44,013 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:44,824 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:45,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:45,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:46,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:48,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:48,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:48,288 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:48,293 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:48,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:49,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:50,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:53,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:53,196 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:53,196 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:53,201 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:53,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:54,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:55,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:56,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:59:56,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:56,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:56,942 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:56,947 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:57,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:58,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:00,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:00,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:00,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:00,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:01,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:01,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:01,556 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:01,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:01,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:04,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:05,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:05,560 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:05,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:05,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:05,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:05,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:08,849 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:10,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:11,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:11,073 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:11,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:11,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:11,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:12,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:14,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:14,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:14,950 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:14,954 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:15,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:15,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:16,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:16,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:19,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:19,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:19,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:19,269 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:19,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:20,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:22,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:23,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:23,844 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:23,844 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:23,849 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:23,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:26,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:00:26,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:28,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:30,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:30,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:30,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:30,006 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:30,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:30,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:30,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:32,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:33,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:34,139 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:34,140 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:34,140 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:34,145 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:34,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:36,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:39,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:42,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:42,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:42,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:42,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:42,880 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:44,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:44,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:45,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:45,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:46,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:46,431 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:46,432 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:46,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:46,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:48,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:50,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:51,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:51,002 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:51,002 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:51,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:51,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:52,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:56,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:56,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:00:56,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:56,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:56,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:56,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:56,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:58,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:00,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:00,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:00,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:01,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:01,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:01,789 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:01,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:01,794 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:01,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:04,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:06,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:10,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:10,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:10,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:10,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:10,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:12,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:12,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:14,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:14,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:14,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:14,013 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:14,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:15,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:16,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:17,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:18,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:18,206 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:18,206 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:18,211 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:18,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:20,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:22,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:22,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:22,250 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:22,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:22,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:23,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:24,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:25,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:25,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:25,942 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:25,947 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:26,219 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:01:26,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:28,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:29,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:30,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:30,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:33,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:33,464 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:33,464 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:33,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:33,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:34,469 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:34,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:38,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:38,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:38,019 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:38,024 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:38,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:40,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:40,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:42,652 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:42,653 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:42,653 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:42,658 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:42,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:44,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:45,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:50,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:51,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:51,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:51,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:51,100 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:51,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:52,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:55,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:55,230 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:55,230 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:55,236 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:55,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:56,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:01:56,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:56,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:58,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:58,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:58,884 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:58,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:58,954 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:00,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:00,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:00,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:00,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:01,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:03,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:03,400 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:03,400 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:03,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:03,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:04,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:06,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:10,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:10,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:10,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:10,744 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:10,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:11,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:12,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:15,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:15,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:15,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:15,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:15,835 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:15,841 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:15,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:16,842 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:18,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:20,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:20,581 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:20,581 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:20,586 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:20,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:22,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:22,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:26,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:02:26,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:26,458 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:26,458 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:26,464 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:26,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:28,464 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:28,987 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:30,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:30,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:30,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:30,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:30,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:30,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:30,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:30,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:32,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:33,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:34,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:34,763 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:34,764 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:34,769 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:34,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:36,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:38,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:38,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:38,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:38,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:38,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:39,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:41,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:41,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:41,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:41,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:41,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:42,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:44,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:45,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:45,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:45,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:46,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:46,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:46,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:46,505 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:47,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:49,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:50,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:50,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:50,658 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:50,658 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:50,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:51,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:53,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:55,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:55,272 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:55,272 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:55,277 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:56,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:56,226 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:02:56,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:57,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:59,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:59,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:59,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:59,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:00,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:00,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:00,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:00,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:01,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:03,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:04,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:04,352 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:04,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:04,358 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:05,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:06,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:07,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:08,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:08,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:08,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:08,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:09,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:11,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:12,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:12,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:12,715 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:12,715 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:12,720 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:13,035 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:15,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:15,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:15,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:16,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:16,449 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:16,450 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:16,454 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:17,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:17,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:19,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:22,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:22,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:22,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:22,925 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:22,930 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:23,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:25,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:26,228 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:03:27,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:27,780 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:27,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:27,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:27,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:28,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:30,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:30,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:30,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:31,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:33,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:36,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:36,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:36,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:36,333 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:37,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:39,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:39,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:40,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:40,759 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:40,759 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:40,764 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:41,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:43,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:44,343 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:44,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:44,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:44,348 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:45,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:45,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:45,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:47,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:49,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:50,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:50,166 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:50,166 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:50,172 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:51,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:53,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:54,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:54,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:54,211 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:54,215 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:55,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:55,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:56,233 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:03:57,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:59,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:59,099 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:59,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:59,103 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:00,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:00,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:00,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:00,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:00,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:01,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:03,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:03,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:03,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:03,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:04,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:05,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:06,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:07,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:07,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:07,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:07,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:08,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:09,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:11,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:11,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:11,572 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:11,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:11,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:12,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:13,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:15,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:15,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:16,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:16,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:16,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:16,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:17,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:17,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:19,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:20,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:20,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:20,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:20,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:21,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:22,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:23,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:26,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:04:28,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:28,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:28,526 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:28,526 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:28,532 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:29,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:30,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:30,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:30,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:31,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:32,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:32,716 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:32,716 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:32,721 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:33,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:33,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:35,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:37,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:37,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:37,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:37,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:38,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:39,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:39,180 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:41,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:41,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:41,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:41,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:42,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:44,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:45,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:45,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:45,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:46,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:46,913 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:46,913 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:46,918 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:47,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:49,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:50,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:52,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:52,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:52,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:52,210 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:53,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:55,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:56,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:56,237 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:04:56,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:56,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:56,696 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:56,701 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:57,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:59,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:00,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:00,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:00,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:01,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:05,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:05,220 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:05,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:05,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:06,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:07,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:07,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:12,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:12,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:12,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:12,542 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:12,547 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:13,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:15,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:15,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:15,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:15,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:17,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:17,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:17,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:17,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:17,315 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:18,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:19,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:21,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:21,249 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:21,249 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:21,254 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:22,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:23,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:23,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:25,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:25,027 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:25,027 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:25,032 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:25,168 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:26,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:05:27,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:28,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:28,783 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:28,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:28,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:28,788 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:29,173 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:30,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:30,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:30,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:31,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:32,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:32,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:32,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:32,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:33,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:34,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:35,180 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:36,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:36,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:36,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:36,314 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:37,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:39,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:40,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:40,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:40,656 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:40,656 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:40,661 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:41,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:43,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:45,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:45,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:45,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:46,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:46,168 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:46,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:46,172 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:46,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:49,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:51,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:51,423 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:51,423 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:51,424 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:51,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:52,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:53,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:56,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:05:56,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:57,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:57,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:57,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:57,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:58,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:59,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:00,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:00,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:00,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:01,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:01,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:01,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:01,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:01,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:02,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:03,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:06,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:08,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:08,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:08,209 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:08,214 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:09,212 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:11,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:12,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:12,030 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:12,030 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:12,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:12,035 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:12,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:15,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:15,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:15,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:15,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:17,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:19,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:19,331 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:19,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:19,336 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:20,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:21,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:22,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:26,240 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:06:27,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:27,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:27,505 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:27,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:27,510 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:28,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:29,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:30,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:30,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:30,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:31,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:31,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:31,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:31,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:31,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:33,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:33,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:38,012 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:38,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:38,013 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:38,018 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:38,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:39,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:41,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:43,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:43,122 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:43,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:43,127 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:43,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:44,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:45,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:45,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:45,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:47,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:47,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:47,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:47,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:48,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:49,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:49,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:54,232 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:54,233 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:54,233 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:54,239 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:54,263 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:55,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:56,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:06:57,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:00,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:00,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:00,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:00,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:00,419 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:00,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:00,419 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:00,425 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:01,269 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:03,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:04,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:04,809 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:04,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:04,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:05,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:05,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:07,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:08,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:08,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:08,776 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:08,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:09,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:11,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:11,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:12,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:12,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:12,544 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:12,549 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:13,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:15,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:15,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:15,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:15,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:16,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:16,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:16,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:16,314 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:17,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:17,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:19,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:21,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:21,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:21,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:21,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:21,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:23,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:23,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:25,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:25,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:25,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:25,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:25,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:26,244 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:07:27,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:28,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:28,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:28,781 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:28,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:28,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:29,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:30,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:30,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:30,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:31,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:33,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:34,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:34,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:34,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:34,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:34,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:37,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:37,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:37,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:37,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:37,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:38,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:38,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:39,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:42,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:42,448 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:42,448 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:42,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:43,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:44,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:45,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:45,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:45,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:45,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:48,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:48,308 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:48,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:48,313 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:48,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:50,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:51,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:53,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:53,444 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:53,444 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:53,449 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:54,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:55,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:55,450 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:56,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:07:57,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:57,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:57,303 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:57,308 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:57,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:59,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:00,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:00,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:00,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:01,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:05,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:05,837 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:05,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:05,842 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:06,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:06,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:07,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:10,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:10,547 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:10,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:10,552 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:11,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:12,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:13,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:15,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:15,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:15,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:16,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:16,041 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:16,041 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:16,046 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:16,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:17,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:18,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:20,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:20,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:20,376 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:20,380 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:21,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:23,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:23,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:25,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:25,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:25,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:25,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:26,249 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:08:26,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:27,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:29,250 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:29,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:29,474 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:29,475 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:29,479 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:30,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:30,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:30,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:30,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:31,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:34,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:35,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:35,559 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:35,559 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:35,564 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:36,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:37,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:39,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:40,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:40,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:40,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:40,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:41,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:43,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:44,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:45,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:45,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:45,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:46,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:46,321 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:46,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:46,326 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:46,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:49,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:49,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:49,636 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:49,636 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:49,641 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:50,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:50,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:51,375 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:53,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:53,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:53,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:53,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:53,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:55,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:56,251 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:08:56,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:57,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:57,852 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:57,852 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:57,856 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:58,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:59,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:00,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:00,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:00,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:01,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:01,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:01,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:01,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:01,639 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:02,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:03,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:05,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:05,333 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:05,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:05,338 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:05,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:07,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:07,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:09,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:09,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:09,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:09,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:10,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:11,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:12,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:14,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:14,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:14,672 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:14,676 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:15,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:15,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:15,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:15,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:17,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:18,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:18,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:18,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:18,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:18,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:19,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:21,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:23,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:23,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:23,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:23,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:23,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:24,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:25,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:26,253 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:09:28,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:28,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:28,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:28,036 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:28,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:29,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:30,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:30,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:30,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:30,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:33,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:33,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:33,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:33,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:33,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:35,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:35,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:38,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:38,290 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:38,291 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:38,296 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:38,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:40,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:41,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:43,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:43,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:43,492 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:43,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:44,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:45,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:45,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:45,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:45,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:45,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:47,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:47,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:47,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:47,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:48,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:49,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:50,896 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:54,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:54,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:54,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:54,373 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:54,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:56,255 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:09:56,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:57,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:58,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:58,955 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:58,955 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:58,960 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:59,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:00,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:00,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:00,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:01,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:01,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:03,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:03,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:03,009 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:03,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:03,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:05,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:07,015 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:07,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:07,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:07,250 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:07,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:07,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:09,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:12,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:13,426 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:13,427 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:13,427 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:13,432 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:13,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:15,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:15,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:15,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:15,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:17,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:17,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:17,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:17,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:17,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:17,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:19,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:21,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:21,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:21,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:21,222 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:21,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:23,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:23,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:25,867 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:25,867 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:25,868 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:25,872 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:26,260 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:10:26,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:27,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:28,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:30,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:30,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:30,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:30,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:30,713 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:30,713 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:30,718 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:31,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:33,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:33,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:34,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:34,402 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:34,403 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:34,407 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:34,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:37,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:38,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:38,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:38,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:38,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:38,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:39,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:41,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:41,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:41,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:41,925 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:41,930 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:42,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:43,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:44,931 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:45,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:45,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:45,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:46,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:46,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:46,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:46,121 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:46,489 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:47,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:49,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:49,963 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:49,963 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:49,963 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:49,968 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:50,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:51,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:54,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:54,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:54,215 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:54,220 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:54,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:55,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:55,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:56,259 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:10:58,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:58,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:58,830 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:58,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:59,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:00,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:00,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:00,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:00,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:01,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:03,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:03,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:03,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:03,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:03,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:05,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:06,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:07,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:07,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:07,831 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:07,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:08,511 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:09,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:11,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:14,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:14,672 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:14,673 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:14,678 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:15,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:15,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:15,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:17,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:17,522 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:19,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:19,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:19,253 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:19,257 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:19,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:21,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:22,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:22,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:22,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:22,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:22,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:23,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:25,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:26,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:11:26,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:26,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:26,662 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:26,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:27,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:28,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:29,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:30,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:30,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:30,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:31,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:31,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:31,404 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:31,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:31,536 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:33,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:34,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:36,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:36,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:36,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:36,794 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:37,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:39,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:39,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:44,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:45,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:45,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:45,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:45,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:45,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:45,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:47,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:49,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:49,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:49,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:49,057 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:49,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:50,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:51,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:53,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:53,227 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:53,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:53,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:53,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:55,232 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:55,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:56,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:12:00,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:00,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:00,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:00,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:01,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:01,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:01,712 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:01,717 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:02,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:03,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:05,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:05,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:05,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:05,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:05,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:06,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:07,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:10,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:10,228 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:10,228 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:10,234 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:10,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:11,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:11,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:13,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:13,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:13,989 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:13,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:14,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:15,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:15,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:15,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:16,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:19,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:19,391 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:19,392 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:19,397 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:19,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:21,398 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:21,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:24,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:24,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:24,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:24,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:24,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:25,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:26,268 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:12:27,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:28,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:28,050 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:28,050 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:28,055 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:28,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:29,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:30,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:30,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:32,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:32,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:32,667 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:32,667 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:32,672 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:33,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:35,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:36,759 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:36,760 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:36,760 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:36,765 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:37,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:37,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:39,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:40,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:40,514 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:40,514 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:40,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:40,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:43,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:43,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:44,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:44,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:44,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:44,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:45,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:45,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:45,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:49,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:51,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:51,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:51,516 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:51,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:51,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:53,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:54,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:56,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:56,171 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:56,171 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:56,176 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:56,267 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:12:56,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:57,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:59,896 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:59,897 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:59,897 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:59,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:59,902 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:00,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:00,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:00,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:00,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:01,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:04,152 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:04,153 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:04,153 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:04,158 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:04,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:05,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:05,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:08,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:08,004 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:08,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:08,009 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:08,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:09,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:11,010 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:13,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:13,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:13,990 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:13,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:14,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:15,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:15,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:15,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:16,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:18,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:18,147 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:18,147 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:18,152 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:18,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:19,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:22,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:22,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:22,405 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:22,405 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:22,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:22,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:25,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:26,272 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:13:27,274 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:28,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:28,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:28,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:28,439 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:28,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:30,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:30,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:31,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:31,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:31,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:31,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:31,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:32,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:32,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:33,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:37,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:37,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:37,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:37,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:37,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:38,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:39,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:43,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:45,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:45,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:45,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:45,535 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:45,535 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:45,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:45,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:47,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:48,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:49,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:49,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:49,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:49,550 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:49,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:51,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:53,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:53,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:53,688 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:53,688 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:53,693 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:54,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:55,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:56,271 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:13:58,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:58,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:58,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:58,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:58,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:59,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:00,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:00,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:00,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:01,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:04,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:04,008 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:04,008 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:04,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:04,013 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:04,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:05,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:08,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:08,094 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:08,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:08,100 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:08,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:09,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:09,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:12,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:12,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:12,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:12,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:12,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:13,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:14,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:15,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:15,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:16,958 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:16,959 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:16,960 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:16,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:17,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:19,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:19,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:21,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:21,339 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:21,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:21,344 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:21,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:23,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:25,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:25,322 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:25,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:25,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:25,327 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:25,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:26,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:14:27,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:29,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:29,823 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:29,823 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:29,828 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:30,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:30,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:30,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:30,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:31,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:35,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:35,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:35,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:35,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:35,391 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:35,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:37,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:39,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:39,534 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:39,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:39,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:39,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:40,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:41,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:44,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:44,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:44,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:44,312 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:44,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:45,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:45,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:45,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:46,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:48,652 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:48,653 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:48,653 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:48,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:48,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:51,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:51,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:56,275 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:14:57,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:00,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:00,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:00,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:00,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:00,374 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:00,375 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:00,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:00,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:01,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:02,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:04,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:04,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:04,686 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:04,691 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:04,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:07,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:07,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:08,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:08,075 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:08,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:08,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:08,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:09,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:12,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:12,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:12,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:12,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:12,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:13,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:13,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:15,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:15,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:17,966 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:17,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:17,967 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:17,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:18,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:18,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:19,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:23,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:26,277 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:15:27,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:27,227 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:27,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:27,232 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:27,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:29,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:29,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:30,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:30,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:31,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:31,228 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:31,228 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:31,233 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:31,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:33,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:34,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:36,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:36,415 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:36,415 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:36,420 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:36,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:37,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:39,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:40,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:40,253 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:40,253 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:40,258 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:40,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:41,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:44,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:44,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:44,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:44,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:44,625 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:44,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:45,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:45,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:47,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:48,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:48,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:48,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:48,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:48,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:49,759 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:51,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:52,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:52,707 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:52,707 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:52,711 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:52,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:55,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:55,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:56,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:15:56,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:56,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:56,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:56,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:57,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:59,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:00,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:00,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:00,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:00,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:00,862 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:00,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:00,863 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:00,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:01,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:03,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:05,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:05,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:05,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:05,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:05,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:06,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:07,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:10,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:10,781 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:10,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:10,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:10,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:11,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:13,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:14,720 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:14,721 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:14,721 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:14,727 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:14,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:15,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:15,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:17,409 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:17,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:20,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:20,198 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:20,198 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:20,202 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:20,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:21,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:23,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:26,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:26,010 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:26,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:26,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:26,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:16:26,824 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:27,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:28,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:29,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:29,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:29,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:29,891 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:30,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:30,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:30,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:31,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:33,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:38,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:38,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:38,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:38,202 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:38,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:39,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:39,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:42,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:42,383 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:42,383 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:42,388 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:42,839 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:43,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:44,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:45,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:45,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:46,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:46,397 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:46,397 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:46,402 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:46,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:47,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:49,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:53,258 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:53,259 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:53,259 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:53,264 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:53,849 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:55,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:55,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:56,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:16:57,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:57,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:57,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:57,853 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:57,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:59,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:00,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:00,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:00,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:00,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:02,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:02,535 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:02,536 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:02,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:02,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:03,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:05,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:08,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:08,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:08,413 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:08,417 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:08,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:09,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:11,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:12,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:12,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:12,870 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:12,875 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:12,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:15,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:15,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:15,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:16,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:16,568 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:16,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:16,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:16,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:16,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:17,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:21,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:24,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:24,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:24,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:24,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:25,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:26,286 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:17:27,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:27,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:30,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:30,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:30,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:30,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:30,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:30,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:30,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:32,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:33,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:34,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:34,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:34,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:34,078 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:34,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:35,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:37,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:37,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:37,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:37,907 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:37,911 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:38,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:39,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:42,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:42,230 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:42,230 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:42,235 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:42,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:43,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:43,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:45,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:45,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:46,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:46,833 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:46,833 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:46,838 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:46,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:48,839 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:49,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:50,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:50,933 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:50,933 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:50,937 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:51,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:53,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:53,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:55,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:55,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:55,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:55,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:55,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:56,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:17:57,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:59,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:59,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:59,519 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:59,520 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:59,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:59,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:00,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:00,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:00,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:01,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:04,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:06,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:06,145 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:06,145 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:06,150 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:06,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:07,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:10,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:12,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:12,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:12,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:12,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:12,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:13,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:15,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:15,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:15,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:15,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:17,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:17,128 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:17,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:17,134 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:17,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:19,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:21,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:21,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:21,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:21,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:21,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:21,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:23,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:25,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:25,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:25,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:25,760 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:25,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:26,290 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:18:26,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:27,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:29,319 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:29,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:29,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:29,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:30,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:30,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:30,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:31,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:31,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:33,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:33,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:33,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:33,325 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:33,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:35,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:36,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:38,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:38,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:38,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:38,057 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:38,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:39,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:42,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:45,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:45,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:45,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:46,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:46,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:46,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:46,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:46,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:47,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:47,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:50,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:50,343 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:50,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:50,348 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:50,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:51,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:53,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:55,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:55,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:55,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:55,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:55,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:56,295 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:18:57,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:59,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:59,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:59,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:59,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:59,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:59,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:00,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:00,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:00,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:01,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:04,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:04,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:04,818 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:04,818 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:04,823 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:04,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:07,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:08,764 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:08,765 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:08,765 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:08,770 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:08,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:09,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:09,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:12,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:12,253 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:12,253 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:12,258 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:12,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:13,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:15,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:15,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:15,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:15,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:17,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:17,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:17,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:17,318 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:17,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:20,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:20,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:21,978 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:21,979 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:21,979 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:21,984 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:22,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:24,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:25,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:25,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:25,805 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:25,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:25,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:26,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:26,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:19:28,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:30,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:30,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:30,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:30,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:30,388 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:30,388 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:30,393 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:31,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:31,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:32,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:34,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:34,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:34,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:34,383 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:35,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:36,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:37,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:38,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:38,878 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:38,878 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:38,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:39,021 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:42,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:42,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:43,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:43,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:43,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:43,374 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:44,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:45,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:45,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:45,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:46,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:47,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:47,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:47,207 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:47,212 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:48,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:48,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:50,033 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:50,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:50,968 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:50,968 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:50,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:51,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:53,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:54,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:56,299 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:19:58,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:58,106 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:58,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:58,111 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:59,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:59,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:00,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:00,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:00,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:00,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:02,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:02,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:02,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:02,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:03,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:04,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:04,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:08,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:08,227 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:08,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:08,232 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:09,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:10,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:10,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:11,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:11,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:11,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:11,998 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:12,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:14,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:15,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:15,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:15,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:15,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:20,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:20,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:20,425 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:20,425 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:20,430 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:21,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:22,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:24,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:24,425 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:24,425 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:24,430 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:25,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:25,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:26,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:26,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:20:27,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:27,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:27,726 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:27,730 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:28,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:30,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:30,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:30,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:30,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:31,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:32,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:32,503 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:32,503 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:32,508 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:33,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:34,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:36,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:36,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:36,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:36,885 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:36,890 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:37,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:40,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:41,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:42,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:42,133 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:42,134 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:42,138 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:43,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:44,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:45,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:45,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:45,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:45,928 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:45,929 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:45,929 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:45,934 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:46,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:46,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:48,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:50,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:50,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:50,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:50,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:50,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:52,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:52,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:54,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:54,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:54,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:54,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:55,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:56,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:56,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:20:57,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:58,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:58,037 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:58,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:58,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:58,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:00,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:00,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:00,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:00,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:02,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:02,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:02,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:02,712 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:02,717 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:03,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:04,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:06,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:06,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:06,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:06,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:07,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:07,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:08,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:10,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:10,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:10,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:10,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:11,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:12,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:13,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:15,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:15,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:15,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:15,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:15,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:15,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:15,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:16,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:18,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:19,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:19,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:19,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:19,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:19,744 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:20,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:22,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:24,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:24,289 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:24,289 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:24,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:25,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:25,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:26,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:26,302 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:21:27,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:27,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:27,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:27,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:28,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:30,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:30,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:30,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:30,128 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:30,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:31,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:31,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:31,778 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:31,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:32,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:34,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:35,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:36,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:36,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:36,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:36,580 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:37,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:38,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:40,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:40,931 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:40,931 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:40,931 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:40,936 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:41,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:42,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:45,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:45,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:45,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:46,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:46,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:46,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:46,139 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:46,144 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:46,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:48,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:49,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:49,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:49,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:49,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:50,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:51,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:52,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:55,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:55,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:55,421 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:55,426 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:56,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:56,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:21:57,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:58,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:59,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:59,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:59,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:59,768 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:00,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:00,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:00,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:00,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:02,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:02,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:05,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:05,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:05,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:05,095 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:05,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:08,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:08,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:13,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:14,035 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:14,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:14,036 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:14,041 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:14,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:15,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:15,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:15,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:16,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:18,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:22,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:22,344 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:22,344 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:22,349 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:23,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:24,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:24,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:26,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:22:26,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:26,314 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:26,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:26,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:27,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:28,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:30,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:30,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:30,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:30,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:30,331 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:30,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:30,336 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:31,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:32,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:34,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:34,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:34,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:34,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:35,187 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:35,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:36,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:39,963 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:39,964 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:39,964 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:39,969 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:40,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:40,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:42,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:45,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:45,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:45,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:45,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:45,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:45,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:45,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:46,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:46,894 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:48,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:51,894 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:54,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:54,226 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:54,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:54,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:55,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:56,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:56,310 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:22:57,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:59,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:59,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:59,976 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:59,981 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:00,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:00,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:00,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:02,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:02,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:04,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:04,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:04,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:04,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:05,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:06,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:07,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:08,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:08,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:08,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:08,440 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:09,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:10,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:12,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:12,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:12,378 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:12,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:13,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:13,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:14,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:15,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:15,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:15,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:16,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:16,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:16,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:16,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:17,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:18,232 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:18,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:21,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:21,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:21,003 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:21,008 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:21,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:22,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:24,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:24,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:24,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:24,866 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:24,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:25,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:26,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:26,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:23:28,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:28,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:28,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:28,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:29,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:29,816 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:30,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:30,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:30,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:33,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:33,187 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:33,187 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:33,192 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:33,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:35,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:36,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:38,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:38,534 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:38,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:38,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:39,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:40,253 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:40,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:42,409 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:42,410 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:42,410 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:42,415 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:43,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:44,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:45,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:45,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:45,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:46,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:47,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:47,129 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:47,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:47,134 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:47,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:50,263 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:51,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:51,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:51,600 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:51,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:51,605 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:52,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:54,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:55,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:55,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:55,728 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:55,732 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:56,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:56,314 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:23:57,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:58,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:59,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:59,894 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:59,894 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:59,899 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:00,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:00,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:00,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:02,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:02,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:03,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:03,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:03,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:03,144 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:03,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:06,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:07,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:07,506 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:07,506 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:07,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:07,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:08,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:10,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:12,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:12,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:12,373 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:12,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:13,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:13,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:14,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:15,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:15,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:15,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:18,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:18,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:18,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:18,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:19,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:19,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:20,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:22,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:22,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:22,765 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:22,770 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:23,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:24,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:24,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:26,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:24:27,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:27,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:27,366 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:27,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:28,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:30,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:30,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:30,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:31,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:31,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:31,659 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:31,664 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:32,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:34,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:35,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:40,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:40,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:40,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:40,991 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:40,996 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:41,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:42,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:44,908 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:44,909 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:44,909 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:44,914 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:45,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:45,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:45,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:45,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:46,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:46,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:51,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:51,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:51,438 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:51,438 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:51,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:52,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:54,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:54,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:54,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:54,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:54,884 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:55,328 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:56,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:24:56,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:57,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:59,056 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:59,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:59,061 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:59,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:00,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:00,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:00,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:02,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:02,808 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:02,809 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:02,809 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:02,814 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:03,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:04,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:06,628 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:06,629 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:06,630 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:06,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:07,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:07,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:08,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:12,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:15,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:15,040 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:15,041 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:15,045 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:15,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:15,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:15,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:15,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:16,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:18,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:20,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:20,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:20,764 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:20,769 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:21,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:22,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:23,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:25,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:25,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:25,481 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:25,487 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:26,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:25:26,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:28,361 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:29,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:29,882 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:29,882 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:29,887 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:30,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:30,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:32,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:34,178 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:34,179 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:34,179 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:34,184 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:34,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:35,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:36,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:40,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:41,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:41,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:41,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:41,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:41,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:42,375 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:44,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:44,985 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:44,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:44,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:45,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:45,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:45,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:45,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:45,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:46,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:49,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:49,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:49,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:49,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:49,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:51,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:52,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:53,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:53,399 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:53,400 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:53,404 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:54,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:56,322 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:25:56,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:56,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:58,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:58,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:58,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:58,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:59,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:00,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:00,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:01,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:06,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:06,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:06,973 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:06,974 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:06,978 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:07,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:08,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:11,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:11,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:11,795 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:11,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:11,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:12,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:14,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:15,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:15,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:15,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:17,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:17,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:17,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:17,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:17,119 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:17,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:18,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:22,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:22,099 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:22,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:22,104 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:22,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:23,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:24,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:26,328 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:26:28,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:29,928 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:29,929 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:29,929 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:29,934 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:30,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:30,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:32,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:33,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:34,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:34,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:34,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:34,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:34,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:36,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:38,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:38,538 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:38,539 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:38,539 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:38,545 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:39,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:40,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:42,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:42,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:42,373 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:42,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:42,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:44,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:44,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:45,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:45,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:45,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:46,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:46,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:46,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:46,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:46,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:48,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:49,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:50,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:50,405 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:50,405 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:50,410 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:50,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:52,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:54,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:54,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:54,475 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:54,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:54,480 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:55,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:56,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:26:56,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:58,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:58,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:58,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:58,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:58,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:00,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:00,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:00,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:00,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:02,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:02,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:02,304 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:02,309 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:02,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:04,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:05,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:06,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:06,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:06,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:06,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:07,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:08,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:10,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:11,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:11,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:11,210 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:11,215 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:11,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:12,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:15,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:15,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:15,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:16,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:16,403 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:16,403 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:16,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:16,408 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:16,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:18,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:20,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:20,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:20,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:20,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:20,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:22,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:22,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:24,935 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:24,936 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:24,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:24,941 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:25,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:26,331 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:27:26,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:27,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:30,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:30,171 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:30,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:30,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:30,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:30,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:32,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:32,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:37,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:37,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:37,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:37,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:37,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:38,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:40,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:42,752 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:45,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:45,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:45,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:46,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:46,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:46,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:46,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:46,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:48,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:48,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:53,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:54,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:54,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:54,599 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:54,604 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:55,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:56,330 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:27:56,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:58,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:58,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:58,607 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:58,607 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:58,612 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:59,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:00,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:00,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:00,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:01,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:01,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:01,884 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:01,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:02,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:03,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:04,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:08,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:10,179 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:10,180 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:10,180 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:10,184 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:10,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:12,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:14,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:14,490 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:14,490 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:14,491 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:14,496 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:14,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:15,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:15,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:15,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:16,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:18,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:18,917 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:18,917 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:18,922 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:19,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:19,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:20,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:22,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:22,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:22,768 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:22,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:23,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:24,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:25,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:26,333 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:28:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:30,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:31,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:31,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:31,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:31,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:31,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:32,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:34,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:36,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:36,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:36,288 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:36,293 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:36,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:37,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:38,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:39,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:39,946 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:39,946 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:39,951 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:40,546 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:42,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:42,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:44,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:44,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:44,682 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:44,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:45,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:45,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:45,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:45,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:46,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:48,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:49,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:49,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:49,186 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:49,190 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:49,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:50,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:53,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:53,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:53,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:53,313 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:53,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:54,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:54,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:56,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:28:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:00,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:00,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:00,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:01,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:01,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:01,627 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:01,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:02,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:04,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:04,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:07,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:07,438 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:07,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:07,443 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:07,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:08,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:10,444 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:12,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:12,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:12,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:12,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:13,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:14,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:15,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:15,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:15,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:16,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:21,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:21,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:21,004 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:21,009 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:21,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:22,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:22,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:25,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:25,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:25,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:25,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:26,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:29:26,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:27,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:28,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:30,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:30,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:31,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:31,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:31,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:31,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:31,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:32,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:33,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:35,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:35,485 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:35,485 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:35,489 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:35,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:38,490 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:38,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:39,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:39,495 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:39,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:39,500 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:39,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:42,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:43,456 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:43,456 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:43,457 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:43,462 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:43,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:44,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:44,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:45,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:45,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:45,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:47,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:47,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:47,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:47,707 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:48,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:49,708 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:50,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:52,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:52,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:52,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:52,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:53,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:54,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:54,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:56,339 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:29:57,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:57,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:57,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:57,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:57,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:58,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:00,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:00,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:00,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:00,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:02,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:02,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:02,774 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:02,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:03,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:04,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:05,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:10,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:11,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:11,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:11,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:11,109 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:11,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:12,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:14,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:14,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:14,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:14,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:15,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:15,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:15,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:15,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:16,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:16,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:19,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:19,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:19,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:19,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:19,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:20,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:22,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:24,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:24,170 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:24,170 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:24,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:24,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:26,341 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:30:26,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:27,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:28,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:28,314 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:28,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:28,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:28,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:30,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:30,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:30,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:33,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:36,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:36,160 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:36,160 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:36,165 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:36,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:38,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:39,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:40,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:40,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:40,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:40,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:40,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:42,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:44,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:45,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:45,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:45,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:45,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:45,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:45,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:45,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:45,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:46,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:49,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:49,084 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:49,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:49,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:49,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:50,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:50,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:53,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:53,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:53,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:53,689 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:54,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:55,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:56,346 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:30:56,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:58,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:58,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:58,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:58,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:58,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:00,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:00,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:00,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:00,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:01,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:02,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:02,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:02,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:02,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:03,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:04,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:06,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:07,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:07,840 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:07,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:07,845 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:08,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:10,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:11,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:12,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:12,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:12,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:12,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:12,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:14,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:15,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:15,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:15,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:17,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:20,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:20,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:20,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:20,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:20,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:22,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:22,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:26,345 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:31:28,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:28,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:28,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:28,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:28,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:29,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:30,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:30,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:30,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:30,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:33,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:33,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:33,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:33,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:33,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:34,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:34,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:38,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:38,909 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:38,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:38,915 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:39,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:39,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:40,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:43,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:43,472 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:43,472 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:43,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:43,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:44,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:45,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:45,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:45,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:45,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:47,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:47,643 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:47,644 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:47,649 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:47,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:50,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:50,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:52,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:52,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:52,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:52,158 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:52,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:54,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:55,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:55,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:55,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:55,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:55,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:56,350 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:31:56,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:58,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:00,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:00,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:00,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:01,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:01,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:01,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:01,923 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:01,928 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:02,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:04,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:06,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:06,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:06,413 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:06,418 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:06,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:07,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:08,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:10,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:10,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:10,404 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:10,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:10,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:12,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:13,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:15,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:15,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:15,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:15,699 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:15,700 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:15,700 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:15,705 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:15,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:18,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:18,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:20,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:20,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:20,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:20,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:21,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:22,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:23,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:26,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:32:26,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:26,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:26,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:26,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:26,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:28,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:29,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:30,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:30,026 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:30,027 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:30,032 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:30,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:30,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:30,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:30,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:32,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:34,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:34,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:34,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:34,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:34,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:35,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:36,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:38,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:38,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:38,795 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:38,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:39,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:40,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:40,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:43,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:43,862 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:43,862 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:43,867 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:44,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:45,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:45,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:45,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:46,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:46,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:47,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:47,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:47,624 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:47,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:47,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:48,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:51,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:51,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:51,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:51,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:51,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:51,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:54,790 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:55,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:55,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:55,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:55,660 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:55,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:56,354 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:32:56,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:57,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:59,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:59,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:59,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:59,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:59,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:00,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:00,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:00,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:00,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:02,430 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:04,259 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:04,260 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:04,260 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:04,265 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:04,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:06,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:08,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:12,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:12,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:12,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:12,342 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:12,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:13,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:14,810 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:15,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:15,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:15,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:15,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:15,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:15,913 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:15,918 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:16,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:18,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:18,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:19,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:19,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:19,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:19,364 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:19,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:20,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:24,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:24,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:24,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:24,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:24,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:24,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:26,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:33:26,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:29,664 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:29,664 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:29,665 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:29,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:29,670 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:29,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:30,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:30,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:30,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:30,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:34,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:34,785 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:34,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:34,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:34,790 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:34,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:36,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:39,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:39,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:39,833 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:39,833 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:39,838 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:40,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:42,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:44,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:44,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:44,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:44,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:44,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:45,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:45,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:45,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:45,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:46,839 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:49,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:49,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:49,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:49,515 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:49,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:50,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:50,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:55,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:55,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:55,348 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:55,353 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:55,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:56,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:56,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:33:56,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:00,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:00,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:00,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:01,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:03,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:03,779 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:03,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:03,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:03,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:04,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:06,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:10,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:10,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:10,726 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:10,731 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:10,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:12,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:12,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:14,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:14,840 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:14,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:14,845 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:14,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:15,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:15,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:15,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:16,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:18,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:20,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:20,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:20,144 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:20,149 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:20,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:22,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:24,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:24,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:24,621 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:24,621 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:24,626 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:24,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:26,357 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:34:26,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:29,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:30,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:30,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:30,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:30,122 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:30,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:30,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:30,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:30,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:32,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:34,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:34,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:34,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:34,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:34,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:35,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:36,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:40,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:40,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:40,256 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:40,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:40,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:41,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:42,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:45,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:45,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:45,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:46,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:48,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:48,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:48,745 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:48,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:48,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:50,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:51,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:52,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:52,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:52,719 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:52,724 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:52,902 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:54,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:56,359 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:34:56,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:56,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:56,727 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:56,732 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:56,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:57,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:58,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:35:00,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:35:00,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:00,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:00,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:00,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:35:00,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:00,853 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:00,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:35:02,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:02,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:35:05,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:05,249 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:35:05,249 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:05,254 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:05,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:35:06,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:35:08,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:13,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:15,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:35:15,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:15,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:18,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:23,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:26,365 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:35:29,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:30,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:35:30,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:30,932 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:35,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:40,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:45,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:50,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:55,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:56,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:36:00,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:03,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:03,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:03,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:05,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:10,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:15,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:18,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:18,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:18,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:20,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:25,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:26,365 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:36:31,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:33,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:33,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:33,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:36,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:41,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:46,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:48,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:48,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:48,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:52,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:56,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:36:58,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:03,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:03,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:03,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:03,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:08,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:13,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:18,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:18,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:18,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:19,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:24,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:26,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:37:30,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:33,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:33,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:33,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:35,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:40,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:45,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:48,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:48,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:48,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:50,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:55,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:56,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:38:01,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:03,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:03,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:03,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:06,647 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:11,648 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:16,648 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:18,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:18,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:18,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:22,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:26,376 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:38:28,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:33,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:33,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:33,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:33,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:38,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:43,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:48,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:48,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:48,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:49,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:54,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:56,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:39:00,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:03,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:39:03,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:39:03,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:39:05,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:10,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:13,300 DEBUG SenderThread:1827008 [sender.py:send():382] send: telemetry +2024-03-12 21:39:13,300 DEBUG SenderThread:1827008 [sender.py:send():382] send: exit +2024-03-12 21:39:13,300 INFO SenderThread:1827008 [sender.py:send_exit():589] handling exit code: 0 +2024-03-12 21:39:13,300 INFO SenderThread:1827008 [sender.py:send_exit():591] handling runtime: 25819 +2024-03-12 21:39:13,305 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:13,306 INFO SenderThread:1827008 [sender.py:send_exit():597] send defer +2024-03-12 21:39:13,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:13,306 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 0 +2024-03-12 21:39:13,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:13,306 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-03-12 21:39:13,306 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 1 +2024-03-12 21:39:13,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:13,306 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 1 +2024-03-12 21:39:13,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:13,306 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-03-12 21:39:13,306 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 2 +2024-03-12 21:39:13,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:13,306 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 2 +2024-03-12 21:39:13,306 INFO HandlerThread:1827008 [system_monitor.py:finish():203] Stopping system monitor +2024-03-12 21:39:13,306 DEBUG SystemMonitor:1827008 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-12 21:39:13,306 DEBUG SystemMonitor:1827008 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-12 21:39:13,309 INFO HandlerThread:1827008 [interfaces.py:finish():202] Joined cpu monitor +2024-03-12 21:39:13,309 INFO HandlerThread:1827008 [interfaces.py:finish():202] Joined disk monitor +2024-03-12 21:39:13,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:39:14,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:39:15,204 INFO HandlerThread:1827008 [interfaces.py:finish():202] Joined gpu monitor +2024-03-12 21:39:15,204 INFO HandlerThread:1827008 [interfaces.py:finish():202] Joined memory monitor +2024-03-12 21:39:15,204 INFO HandlerThread:1827008 [interfaces.py:finish():202] Joined network monitor +2024-03-12 21:39:15,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:39:15,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:15,205 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-03-12 21:39:15,205 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 3 +2024-03-12 21:39:15,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:39:15,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:39:15,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:15,206 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 3 +2024-03-12 21:39:15,206 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:39:15,206 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:15,211 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:15,211 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:15,211 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-03-12 21:39:15,211 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 4 +2024-03-12 21:39:15,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:15,211 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 4 +2024-03-12 21:39:15,211 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:15,211 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-03-12 21:39:15,211 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 5 +2024-03-12 21:39:15,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:15,211 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 5 +2024-03-12 21:39:15,212 DEBUG SenderThread:1827008 [sender.py:send():382] send: summary +2024-03-12 21:39:15,216 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:15,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:15,216 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-03-12 21:39:15,216 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 6 +2024-03-12 21:39:15,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:15,216 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 6 +2024-03-12 21:39:15,217 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:15,217 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-03-12 21:39:15,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:15,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:39:15,435 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 7 +2024-03-12 21:39:15,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:39:15,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:15,435 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 7 +2024-03-12 21:39:15,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:15,435 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-03-12 21:39:15,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/config.yaml +2024-03-12 21:39:15,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:39:15,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:39:16,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:39:18,361 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 8 +2024-03-12 21:39:18,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:39:18,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:18,362 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 8 +2024-03-12 21:39:18,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:18,362 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-03-12 21:39:18,362 INFO SenderThread:1827008 [job_builder.py:build():296] Attempting to build job artifact +2024-03-12 21:39:18,363 INFO SenderThread:1827008 [job_builder.py:_get_source_type():426] is repo sourced job +2024-03-12 21:39:18,438 INFO SenderThread:1827008 [job_builder.py:build():402] adding wandb-job metadata file +2024-03-12 21:39:18,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:39:18,470 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 9 +2024-03-12 21:39:18,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:18,471 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 9 +2024-03-12 21:39:18,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: artifact +2024-03-12 21:39:19,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:39:19,483 INFO SenderThread:1827008 [sender.py:send_artifact():1494] sent artifact job-https___github.com_TRI-ML_prismatic-vlms.git_scripts_pretrain.py - {'id': 'QXJ0aWZhY3Q6NzUzNTUzNzEx', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE0MzcwMzk1OA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NzUzNTUzNzEx', 'versionIndex': 7}}} +2024-03-12 21:39:19,483 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:19,483 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-03-12 21:39:19,483 INFO SenderThread:1827008 [dir_watcher.py:finish():358] shutting down directory watcher +2024-03-12 21:39:20,467 INFO SenderThread:1827008 [dir_watcher.py:finish():388] scan: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files +2024-03-12 21:39:20,468 INFO SenderThread:1827008 [dir_watcher.py:finish():402] scan save: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/config.yaml config.yaml +2024-03-12 21:39:20,468 INFO SenderThread:1827008 [dir_watcher.py:finish():402] scan save: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/conda-environment.yaml conda-environment.yaml +2024-03-12 21:39:20,468 INFO SenderThread:1827008 [dir_watcher.py:finish():402] scan save: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-metadata.json wandb-metadata.json +2024-03-12 21:39:20,471 INFO SenderThread:1827008 [dir_watcher.py:finish():402] scan save: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/requirements.txt requirements.txt +2024-03-12 21:39:20,471 INFO SenderThread:1827008 [dir_watcher.py:finish():402] scan save: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log output.log +2024-03-12 21:39:20,472 INFO SenderThread:1827008 [dir_watcher.py:finish():402] scan save: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json wandb-summary.json +2024-03-12 21:39:20,472 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 10 +2024-03-12 21:39:20,472 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:39:20,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:20,475 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 10 +2024-03-12 21:39:20,477 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:20,478 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-03-12 21:39:20,478 INFO SenderThread:1827008 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 21:39:21,180 INFO wandb-upload_0:1827008 [upload_job.py:push():131] Uploaded file runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/config.yaml +2024-03-12 21:39:21,246 INFO wandb-upload_2:1827008 [upload_job.py:push():131] Uploaded file runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/requirements.txt +2024-03-12 21:39:21,275 INFO wandb-upload_4:1827008 [upload_job.py:push():131] Uploaded file runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:39:21,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:39:21,277 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:39:21,279 INFO wandb-upload_1:1827008 [upload_job.py:push():131] Uploaded file runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/conda-environment.yaml +2024-03-12 21:39:21,292 INFO wandb-upload_3:1827008 [upload_job.py:push():131] Uploaded file runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:39:21,492 INFO Thread-11 (_thread_body):1827008 [sender.py:transition_state():617] send defer: 11 +2024-03-12 21:39:21,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:21,493 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 11 +2024-03-12 21:39:21,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:21,493 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-03-12 21:39:21,493 INFO SenderThread:1827008 [file_pusher.py:join():178] waiting for file pusher +2024-03-12 21:39:21,493 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 12 +2024-03-12 21:39:21,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:21,494 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 12 +2024-03-12 21:39:21,494 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:21,494 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-03-12 21:39:21,494 INFO SenderThread:1827008 [file_stream.py:finish():595] file stream finish called +2024-03-12 21:39:22,119 INFO SenderThread:1827008 [file_stream.py:finish():599] file stream finish is done +2024-03-12 21:39:22,119 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 13 +2024-03-12 21:39:22,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:22,119 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 13 +2024-03-12 21:39:22,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:22,119 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-03-12 21:39:22,119 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 14 +2024-03-12 21:39:22,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:22,120 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 14 +2024-03-12 21:39:22,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: final +2024-03-12 21:39:22,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: footer +2024-03-12 21:39:22,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:22,120 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-03-12 21:39:22,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:39:22,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:39:22,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:39:22,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: server_info +2024-03-12 21:39:22,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: server_info +2024-03-12 21:39:22,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: get_summary +2024-03-12 21:39:22,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: sampled_history +2024-03-12 21:39:22,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: job_info +2024-03-12 21:39:22,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: job_info +2024-03-12 21:39:22,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: shutdown +2024-03-12 21:39:22,324 INFO HandlerThread:1827008 [handler.py:finish():869] shutting down handler +2024-03-12 21:39:23,165 INFO WriterThread:1827008 [datastore.py:close():296] close: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/run-zo3nkbji.wandb +2024-03-12 21:39:23,324 INFO SenderThread:1827008 [sender.py:finish():1572] shutting down sender +2024-03-12 21:39:23,324 INFO SenderThread:1827008 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 21:39:23,324 INFO SenderThread:1827008 [file_pusher.py:join():178] waiting for file pusher